Install Elasticsearch with Docker

Hello World!

I'm trying to follow https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-compose-file, I copy .env file, change password, then copy and paste docker-compose.yml and then the rest:

root@dos:/opt/elk# docker-compose down
root@dos:/opt/elk# for volume in elk_certs elk_esdata01 elk_esdata02 elk_esdata03 elk_kibanadata ; do docker volume rm $volume ; done
Error response from daemon: volume elk_certs not found
Error response from daemon: volume elk_esdata01 not found
Error response from daemon: volume elk_esdata02 not found
Error response from daemon: volume elk_esdata03 not found
Error response from daemon: volume elk_kibanadata not found
root@dos:/opt/elk# date
Mon Sep  4 14:20:02 UTC 2023
root@dos:/opt/elk# docker-compose up -d
[+] Running 11/11
 ✔ Network elk_default      Created                                                                                                                                                                        0.1s
 ✔ Volume "elk_kibanadata"  Created                                                                                                                                                                        0.0s
 ✔ Volume "elk_certs"       Created                                                                                                                                                                        0.0s
 ✔ Volume "elk_esdata01"    Created                                                                                                                                                                        0.0s
 ✔ Volume "elk_esdata02"    Created                                                                                                                                                                        0.0s
 ✔ Volume "elk_esdata03"    Created                                                                                                                                                                        0.0s
 ✔ Container elk-setup-1    Healthy                                                                                                                                                                        5.0s
 ✔ Container elk-es01-1     Healthy                                                                                                                                                                       46.4s
 ✔ Container elk-es02-1     Healthy                                                                                                                                                                       36.4s
 ✔ Container elk-es03-1     Healthy                                                                                                                                                                       37.4s
 ✔ Container elk-kibana-1   Started                                                                                                                                                                       46.8s
root@dos:/opt/elk# docker-compose ps
NAME                IMAGE                                                 COMMAND                  SERVICE             CREATED             STATUS                            PORTS
elk-es01-1          docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   es01                50 seconds ago      Up 44 seconds (healthy)           127.0.0.1:9200->9200/tcp, 9300/tcp
elk-es02-1          docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   es02                50 seconds ago      Up 43 seconds (healthy)           9200/tcp, 9300/tcp
elk-es03-1          docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   es03                50 seconds ago      Up 42 seconds (healthy)           9200/tcp, 9300/tcp
elk-kibana-1        docker.elastic.co/kibana/kibana:8.9.1                 "/bin/tini -- /usr/l…"   kibana              50 seconds ago      Up 2 seconds (health: starting)   0.0.0.0:5601->5601/tcp, :::5601->5601/tcp
root@dos:/opt/elk# date
Mon Sep  4 14:21:58 UTC 2023
root@dos:/opt/elk# docker-compose ps
NAME                IMAGE                                   COMMAND                  SERVICE             CREATED              STATUS                    PORTS
elk-kibana-1        docker.elastic.co/kibana/kibana:8.9.1   "/bin/tini -- /usr/l…"   kibana              About a minute ago   Up 54 seconds (healthy)   0.0.0.0:5601->5601/tcp, :::5601->5601/tcp
root@dos:/opt/elk# docker ps -a | grep docker.elastic.co
1988bf84900b   docker.elastic.co/kibana/kibana:8.9.1                 "/bin/tini -- /usr/l…"   2 minutes ago    Up About a minute (healthy)     0.0.0.0:5601->5601/tcp, :::5601->5601/tcp                                                           elk-kibana-1
5c3cdc56e8bf   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   2 minutes ago    Exited (137) 48 seconds ago                                                                                                         elk-es03-1
540502bfdd15   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   2 minutes ago    Exited (137) 45 seconds ago                                                                                                         elk-es02-1
e88e560843a4   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   2 minutes ago    Exited (137) 40 seconds ago                                                                                                         elk-es01-1
addd3980d874   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   2 minutes ago    Exited (0) About a minute ago                                                                                                       elk-setup-1
root@dos:/opt/elk# date
Mon Sep  4 14:27:20 UTC 2023
root@dos:/opt/elk# docker-compose ps
NAME                IMAGE               COMMAND             SERVICE             CREATED             STATUS              PORTS
root@dos:/opt/elk# docker ps -a | grep docker.elastic.co
1988bf84900b   docker.elastic.co/kibana/kibana:8.9.1                 "/bin/tini -- /usr/l…"   7 minutes ago    Exited (1) 18 seconds ago                                                                                                        elk-kibana-1
5c3cdc56e8bf   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   7 minutes ago    Exited (137) 5 minutes ago                                                                                                       elk-es03-1
540502bfdd15   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   7 minutes ago    Exited (137) 5 minutes ago                                                                                                       elk-es02-1
e88e560843a4   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   7 minutes ago    Exited (137) 5 minutes ago                                                                                                       elk-es01-1
addd3980d874   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   7 minutes ago    Exited (0) 6 minutes ago                                                                                                         elk-setup-1
root@dos:/opt/elk#

after a while elk completely fails to start..

Please advise!
Thank you in advance :wink:

docker logs:

root@dos:/opt/elk# docker ps -a | grep docker.elastic.co
1988bf84900b   docker.elastic.co/kibana/kibana:8.9.1                 "/bin/tini -- /usr/l…"   10 minutes ago   Exited (1) 3 minutes ago                                                                                                         elk-kibana-1
5c3cdc56e8bf   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   10 minutes ago   Exited (137) 8 minutes ago                                                                                                       elk-es03-1
540502bfdd15   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   10 minutes ago   Exited (137) 8 minutes ago                                                                                                       elk-es02-1
e88e560843a4   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   10 minutes ago   Exited (137) 8 minutes ago                                                                                                       elk-es01-1
addd3980d874   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   10 minutes ago   Exited (0) 9 minutes ago                                                                                                         elk-setup-1
root@dos:/opt/elk# docker logs addd3980d874
Creating CA
Archive:  config/certs/ca.zip
   creating: config/certs/ca/
  inflating: config/certs/ca/ca.crt
  inflating: config/certs/ca/ca.key
Creating certs
Archive:  config/certs/certs.zip
   creating: config/certs/es01/
  inflating: config/certs/es01/es01.crt
  inflating: config/certs/es01/es01.key
   creating: config/certs/es02/
  inflating: config/certs/es02/es02.crt
  inflating: config/certs/es02/es02.key
   creating: config/certs/es03/
  inflating: config/certs/es03/es03.crt
  inflating: config/certs/es03/es03.key
Setting file permissions
Waiting for Elasticsearch availability
Setting kibana_system password
All done!
root@dos:/opt/elk# docker logs e88e560843a4 | grep -v '"INFO"'
Created elasticsearch keystore in /usr/share/elasticsearch/config/elasticsearch.keystore
{"@timestamp":"2023-09-04T14:21:44.432Z", "log.level": "WARN", "message":"failed to connect to {es02}{0QnyKcWaRgyJNiQvtTKwHA}{dbbSVXbHTjKBo9DfBbXLyA}{es02}{172.30.0.4}{172.30.0.4:9300}{cdfhilmrstw}{8.9.1}{xpack.installed=true, ml.machine_memory=1073741824, ml.allocated_processors=8, ml.max_jvm_size=536870912, ml.allocated_processors_double=8.0} (tried [1] times)", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[es01][generic][T#2]","log.logger":"org.elasticsearch.cluster.NodeConnectionsService","elasticsearch.cluster.uuid":"JeWz8oRTSRqjnntRmtNPQg","elasticsearch.node.id":"mAZ2Xh0rRaa4EwhT0GHR_w","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei","error.type":"org.elasticsearch.transport.ConnectTransportException","error.message":"[es02][172.30.0.4:9300] connect_exception","error.stack_trace":"org.elasticsearch.transport.ConnectTransportException: [es02][172.30.0.4:9300] connect_exception\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:1144)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.SubscribableListener$FailureResult.complete(SubscribableListener.java:285)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.SubscribableListener.tryComplete(SubscribableListener.java:197)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.SubscribableListener.addListener(SubscribableListener.java:96)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.SubscribableListener.addListener(SubscribableListener.java:71)\n\tat org.elasticsearch.transport.netty4@8.9.1/org.elasticsearch.transport.netty4.Netty4TcpChannel.addConnectListener(Netty4TcpChannel.java:139)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.TcpTransport.initiateConnection(TcpTransport.java:422)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:384)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.ClusterConnectionManager.internalOpenConnection(ClusterConnectionManager.java:364)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.ClusterConnectionManager.connectToNodeOrRetry(ClusterConnectionManager.java:212)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.ClusterConnectionManager.connectToNode(ClusterConnectionManager.java:111)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:491)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:471)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.NodeConnectionsService$ConnectionTarget.doConnect(NodeConnectionsService.java:288)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.NodeConnectionsService$ConnectionTarget.lambda$connect$0(NodeConnectionsService.java:235)\n\tat java.base/java.util.ArrayList.forEach(ArrayList.java:1511)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.NodeConnectionsService.connectToNodes(NodeConnectionsService.java:124)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.ClusterApplierService.connectToNodesAsync(ClusterApplierService.java:518)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.ClusterApplierService.connectToNodesAndWait(ClusterApplierService.java:508)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:482)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:426)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:154)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:916)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:217)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:183)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/java.lang.Thread.run(Thread.java:1623)\nCaused by: org.elasticsearch.common.util.concurrent.UncategorizedExecutionException: Failed execution\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.SubscribableListener.wrapAsExecutionException(SubscribableListener.java:178)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.ListenableFuture.wrapException(ListenableFuture.java:38)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.ListenableFuture.wrapException(ListenableFuture.java:27)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.SubscribableListener.onFailure(SubscribableListener.java:141)\n\tat org.elasticsearch.transport.netty4@8.9.1/org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:61)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)\n\tat org.elasticsearch.security@8.9.1/org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport$ClientSslHandlerInitializer.lambda$connect$1(SecurityNetty4Transport.java:289)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:689)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:652)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)\n\tat io.netty.common@4.1.94.Final/io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\t... 1 more\nCaused by: java.util.concurrent.ExecutionException: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 172.30.0.4/172.30.0.4:9300\n\t... 29 more\nCaused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 172.30.0.4/172.30.0.4:9300\nCaused by: java.net.ConnectException: Connection refused\n\tat java.base/sun.nio.ch.Net.pollConnect(Native Method)\n\tat java.base/sun.nio.ch.Net.pollConnectNow(Net.java:673)\n\tat java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:973)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:689)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:652)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)\n\tat io.netty.common@4.1.94.Final/io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat java.base/java.lang.Thread.run(Thread.java:1623)\n"}

ERROR: Elasticsearch exited unexpectedly
root@dos:/opt/elk# docker logs 540502bfdd15 | grep -v '"INFO"'
{"@timestamp":"2023-09-04T14:21:41.700Z", "log.level": "WARN", "message":"failed to connect to {es03}{ROD5xYFqS9iiqaMN9imr5g}{DUZCubCsT6GCJAZ-KcrQww}{es03}{172.30.0.5}{172.30.0.5:9300}{cdfhilmrstw}{8.9.1}{ml.allocated_processors=8, ml.machine_memory=1073741824, xpack.installed=true, ml.allocated_processors_double=8.0, ml.max_jvm_size=536870912} (tried [1] times)", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[es02][generic][T#3]","log.logger":"org.elasticsearch.cluster.NodeConnectionsService","trace.id":"dd2a386b32268cb337a755c3c5b74cc6","elasticsearch.cluster.uuid":"JeWz8oRTSRqjnntRmtNPQg","elasticsearch.node.id":"0QnyKcWaRgyJNiQvtTKwHA","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"kalei","error.type":"org.elasticsearch.transport.ConnectTransportException","error.message":"[es03][172.30.0.5:9300] connect_exception","error.stack_trace":"org.elasticsearch.transport.ConnectTransportException: [es03][172.30.0.5:9300] connect_exception\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:1144)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.SubscribableListener$FailureResult.complete(SubscribableListener.java:285)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.SubscribableListener.tryComplete(SubscribableListener.java:197)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.SubscribableListener.addListener(SubscribableListener.java:96)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.SubscribableListener.addListener(SubscribableListener.java:71)\n\tat org.elasticsearch.transport.netty4@8.9.1/org.elasticsearch.transport.netty4.Netty4TcpChannel.addConnectListener(Netty4TcpChannel.java:139)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.TcpTransport.initiateConnection(TcpTransport.java:422)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:384)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.ClusterConnectionManager.internalOpenConnection(ClusterConnectionManager.java:364)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.ClusterConnectionManager.connectToNodeOrRetry(ClusterConnectionManager.java:212)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.ClusterConnectionManager.connectToNode(ClusterConnectionManager.java:111)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:491)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:471)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.NodeConnectionsService$ConnectionTarget.doConnect(NodeConnectionsService.java:288)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.NodeConnectionsService$ConnectionTarget.lambda$connect$0(NodeConnectionsService.java:235)\n\tat java.base/java.util.ArrayList.forEach(ArrayList.java:1511)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.NodeConnectionsService.connectToNodes(NodeConnectionsService.java:124)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.ClusterApplierService.connectToNodesAsync(ClusterApplierService.java:518)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.ClusterApplierService.connectToNodesAndWait(ClusterApplierService.java:508)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:482)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:426)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:154)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:916)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:217)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:183)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/java.lang.Thread.run(Thread.java:1623)\nCaused by: org.elasticsearch.common.util.concurrent.UncategorizedExecutionException: Failed execution\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.SubscribableListener.wrapAsExecutionException(SubscribableListener.java:178)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.ListenableFuture.wrapException(ListenableFuture.java:38)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.ListenableFuture.wrapException(ListenableFuture.java:27)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.SubscribableListener.onFailure(SubscribableListener.java:141)\n\tat org.elasticsearch.transport.netty4@8.9.1/org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:61)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)\n\tat org.elasticsearch.security@8.9.1/org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport$ClientSslHandlerInitializer.lambda$connect$1(SecurityNetty4Transport.java:289)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:689)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:652)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)\n\tat io.netty.common@4.1.94.Final/io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\t... 1 more\nCaused by: java.util.concurrent.ExecutionException: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 172.30.0.5/172.30.0.5:9300\n\t... 29 more\nCaused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 172.30.0.5/172.30.0.5:9300\nCaused by: java.net.ConnectException: Connection refused\n\tat java.base/sun.nio.ch.Net.pollConnect(Native Method)\n\tat java.base/sun.nio.ch.Net.pollConnectNow(Net.java:673)\n\tat java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:973)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:689)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:652)\n\tat io.netty.transport@4.1.94.Final/io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)\n\tat io.netty.common@4.1.94.Final/io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)\n\tat io.netty.common@4.1.94.Final/io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat java.base/java.lang.Thread.run(Thread.java:1623)\n"}

ERROR: Elasticsearch exited unexpectedly
{"@timestamp":"2023-09-04T14:21:41.974Z", "log.level": "WARN", "message":"[[.kibana_ingest_8.9.1_001][0]] failed to perform indices:data/write/bulk[s] on replica [.kibana_ingest_8.9.1_001][0], node[ROD5xYFqS9iiqaMN9imr5g], [R], s[STARTED], a[id=8hoMUdLwQSegUe9tf12Q9A], failed_attempts[0]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[es02][scheduler][T#1]","log.logger":"org.elasticsearch.action.bulk.TransportShardBulkAction","trace.id":"639b67e25648f1577ed8476a42364645","elasticsearch.cluster.uuid":"JeWz8oRTSRqjnntRmtNPQg","elasticsearch.node.id":"0QnyKcWaRgyJNiQvtTKwHA","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"kalei","error.type":"org.elasticsearch.client.internal.transport.NoNodeAvailableException","error.message":"unknown node [ROD5xYFqS9iiqaMN9imr5g]","error.stack_trace":"org.elasticsearch.client.internal.transport.NoNodeAvailableException: unknown node [ROD5xYFqS9iiqaMN9imr5g]\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1239)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:301)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:131)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:983)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.threadpool.ThreadPool$1.run(ThreadPool.java:442)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:577)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)\n\tat java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/java.lang.Thread.run(Thread.java:1623)\n\tSuppressed: org.elasticsearch.transport.NodeNotConnectedException: [es03][172.30.0.5:9300] Node not connected\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:283)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.TransportService.getConnection(TransportService.java:856)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:753)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1253)\n\t\t... 11 more\n\tSuppressed: org.elasticsearch.transport.NodeNotConnectedException: [es03][172.30.0.5:9300] Node not connected\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.ClusterConnectionManager.getConnection(ClusterConnectionManager.java:283)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.TransportService.getConnection(TransportService.java:856)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:753)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:1253)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.replication.ReplicationOperation$3.tryAction(ReplicationOperation.java:301)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.RetryableAction$1.doRun(RetryableAction.java:131)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:237)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.RetryableAction.run(RetryableAction.java:109)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.replication.ReplicationOperation.performOnReplica(ReplicationOperation.java:320)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.replication.ReplicationOperation.performOnReplicas(ReplicationOperation.java:224)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.replication.ReplicationOperation.handlePrimaryResult(ReplicationOperation.java:171)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.ActionListener$2.onResponse(ActionListener.java:169)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:319)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.bulk.TransportShardBulkAction$2.finishRequest(TransportShardBulkAction.java:273)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.bulk.TransportShardBulkAction$2.doRun(TransportShardBulkAction.java:235)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.bulk.TransportShardBulkAction.performOnPrimary(TransportShardBulkAction.java:286)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:137)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.bulk.TransportShardBulkAction.dispatchedShardOperationOnPrimary(TransportShardBulkAction.java:74)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.support.replication.TransportWriteAction$1.doRun(TransportWriteAction.java:215)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:983)\n\t\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)\n\t\t... 3 more\n"}
{"@timestamp":"2023-09-04T14:21:43.986Z", "log.level": "WARN", "message":"[.security-7][0] marking unavailable shards as stale: [eNhBjyqtSu-nrpL2UvvWGg]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[es02][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.routing.allocation.AllocationService","trace.id":"dd2a386b32268cb337a755c3c5b74cc6","elasticsearch.cluster.uuid":"JeWz8oRTSRqjnntRmtNPQg","elasticsearch.node.id":"0QnyKcWaRgyJNiQvtTKwHA","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-04T14:21:43.987Z", "log.level": "WARN", "message":"[.kibana_ingest_8.9.1_001][0] marking unavailable shards as stale: [8hoMUdLwQSegUe9tf12Q9A]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[es02][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.routing.allocation.AllocationService","trace.id":"dd2a386b32268cb337a755c3c5b74cc6","elasticsearch.cluster.uuid":"JeWz8oRTSRqjnntRmtNPQg","elasticsearch.node.id":"0QnyKcWaRgyJNiQvtTKwHA","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-04T14:21:43.987Z", "log.level": "WARN", "message":"[.kibana_security_solution_8.9.1_001][0] marking unavailable shards as stale: [6qx_bPrOR1ynt1ZlpI0dug]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[es02][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.routing.allocation.AllocationService","trace.id":"dd2a386b32268cb337a755c3c5b74cc6","elasticsearch.cluster.uuid":"JeWz8oRTSRqjnntRmtNPQg","elasticsearch.node.id":"0QnyKcWaRgyJNiQvtTKwHA","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-04T14:21:43.988Z", "log.level": "WARN", "message":"[.kibana_alerting_cases_8.9.1_001][0] marking unavailable shards as stale: [e_YjKXF2RMyYPxiMv_U3Ow]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[es02][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.routing.allocation.AllocationService","trace.id":"dd2a386b32268cb337a755c3c5b74cc6","elasticsearch.cluster.uuid":"JeWz8oRTSRqjnntRmtNPQg","elasticsearch.node.id":"0QnyKcWaRgyJNiQvtTKwHA","elasticsearch.node.name":"es02","elasticsearch.cluster.name":"kalei"}
root@dos:/opt/elk# docker logs 5c3cdc56e8bf | grep -v '"INFO"'

{"@timestamp":"2023-09-04T14:20:56.731Z", "log.level": "WARN", "message":"failing [elected-as-master ([2] nodes joined in term 1)[_FINISH_ELECTION_, {es03}{ROD5xYFqS9iiqaMN9imr5g}{DUZCubCsT6GCJAZ-KcrQww}{es03}{172.30.0.5}{172.30.0.5:9300}{cdfhilmrstw}{8.9.1} completing election, {es02}{0QnyKcWaRgyJNiQvtTKwHA}{dbbSVXbHTjKBo9DfBbXLyA}{es02}{172.30.0.4}{172.30.0.4:9300}{cdfhilmrstw}{8.9.1} completing election]]: failed to commit cluster state version [1]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[es03][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.service.MasterService","elasticsearch.node.name":"es03","elasticsearch.cluster.name":"kalei","error.type":"org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException","error.message":"node is no longer master for term 1 while handling publication","error.stack_trace":"org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: node is no longer master for term 1 while handling publication\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.coordination.Coordinator.publish(Coordinator.java:1523)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.MasterService.publish(MasterService.java:475)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.MasterService.publishClusterStateUpdate(MasterService.java:353)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.MasterService.lambda$executeAndPublishBatch$2(MasterService.java:292)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.ActionListener.run(ActionListener.java:382)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.MasterService.executeAndPublishBatch(MasterService.java:274)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.MasterService$BatchingTaskQueue$Processor.lambda$run$2(MasterService.java:1622)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.ActionListener.run(ActionListener.java:382)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.MasterService$BatchingTaskQueue$Processor.run(MasterService.java:1619)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.MasterService$5.lambda$doRun$0(MasterService.java:1237)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.action.ActionListener.run(ActionListener.java:382)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.cluster.service.MasterService$5.doRun(MasterService.java:1216)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:983)\n\tat org.elasticsearch.server@8.9.1/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)\n\tat java.base/ERROR: Elasticsearch exited unexpectedly
java.lang.Thread.run(Thread.java:1623)\n"}
root@dos:/opt/elk#

As per these logs, es02 & es03 are not available and failed to connect.
Please make sure these nodes are up and then try again.

Please provide complete elastic logs.

Also, have you set these values in your docker

@HiteshSingh - thank you for looking into my thread) yes, I did set vm.max_map_count to 262144 (sorry for not mention this in my initial question).

root@dos:~# cp /var/lib/docker/containers/addd3980d874baa84b8f4c950934583186a6402518ac2f517b324c1ed559ed24/addd3980d874baa84b8f4c950934583186a6402518ac2f517b324c1ed559ed24-json.log elk-setup-1.log
root@dos:~# cp /var/lib/docker/containers/e88e560843a4ae9356f32d4aa1f7141b5b8abfbb3f8a6fca4a91e38ca8468cff/e88e560843a4ae9356f32d4aa1f7141b5b8abfbb3f8a6fca4a91e38ca8468cff-json.log elk-es01-1.log
root@dos:~# cp /var/lib/docker/containers/540502bfdd15a6c60c2d0de7d2bd00e82a926c8c4310af5fa8a2c35789ec9a5d/540502bfdd15a6c60c2d0de7d2bd00e82a926c8c4310af5fa8a2c35789ec9a5d-json.log elk-es02-1.log
root@dos:~# cp /var/lib/docker/containers/5c3cdc56e8bf85bbdd52b4636438ae2fb22ac7a19b67818460702ec142009f9d/5c3cdc56e8bf85bbdd52b4636438ae2fb22ac7a19b67818460702ec142009f9d-json.log elk-es03-1.log
root@dos:~# cp /var/lib/docker/containers/1988bf84900bb4725700e3b5c25c88879bc5dbbdfcdc681b849e2f80d9c668b1/1988bf84900bb4725700e3b5c25c88879bc5dbbdfcdc681b849e2f80d9c668b1-json.log elk-kibana-1.log
root@dos:~# tar -zcf elk_logs.tgz elk-*
root@dos:~# shasum elk_logs.tgz
434b47c5da85d19a13bc1dd2835223d048d049b4  elk_logs.tgz
root@dos:~#

elk_logs.tgz:

As per the logs, your master node is es02 and once that is unavailable other nodes cannot form a cluster.

Can you also share your elasticsearch.yml files for all the 3 nodes.

Meanwhile, check this out as well under Elasticsearch cluster configurations for production

@HiteshSingh i'm using elasaticsearch.yml that comes out of the box which is being shipped with the image a copy of that file is within the official image.

thank you for another elasticsearch tutorial, however this thread about official set up / installation of elasticsearch stack via docker :wink:

as you see in my initial question: nodes are dyeing one after another hence me asking this question, i don't understand why they aren't staying up..

Did you try giving max memory docker can use

docker run --name es01 -m 4GB -p 9200:9200 -it docker.elastic.co/elasticsearch/elasticsearch:8.6.2

@HiteshSingh no, i did not :wink: as it doesn't look like it's failing due to insufficient memory (i believe the default value out of the box is 2gb and the way i bring elk up is simple by issuing docker-compose up -d.

Hi @alexus

Assuming you followed the exact directions in the documents... Sometimes the cluster may not come up just for whatever reason.

And that really is just a sample. And the problem is if it does not come up the first time you can try running it over and over and over again and it will not come up.

What I have found you need to do is completely clean up, shut down and prune all the mounts and everything and then try again.

We've had several users have the same experience and this is how they got it to work.

Take a look at this

hi @stephenb , thanks for looking into my thread :wink:

i did follow directions exactly and even included steps that i took for completely cleaning up an environment before re-running it again and was able to replicate same undesirable behavior every single time and at this point not sure what to do other then ask my question here..

root@dos:/opt/elk# docker system prune --all
WARNING! This will remove:
  - all stopped containers
  - all networks not used by at least one container
  - all images without at least one container associated to them
  - all build cache

Are you sure you want to continue? [y/N] y
Total reclaimed space: 0B
root@dos:/opt/elk# docker-compose up -d
[+] Running 27/27
 ✔ es01 Pulled                                                                                                                                                                                            19.2s
 ✔ es03 10 layers [⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿]      0B/0B      Pulled                                                                                                                                                     19.2s
   ✔ f9175e7b73a4 Already exists                                                                                                                                                                           0.0s
   ✔ 7ceecf917bdd Already exists                                                                                                                                                                           0.6s
   ✔ 5722447c4365 Already exists                                                                                                                                                                           0.6s
   ✔ 89732bc75041 Pull complete                                                                                                                                                                            1.6s
   ✔ 77b7fd63f55a Pull complete                                                                                                                                                                           16.1s
   ✔ 6fd9ea970a54 Pull complete                                                                                                                                                                           16.1s
   ✔ 9c46f3874f5e Pull complete                                                                                                                                                                           16.1s
   ✔ ae01b9c7dd90 Pull complete                                                                                                                                                                           16.2s
   ✔ 7d8705bc5737 Pull complete                                                                                                                                                                           16.2s
   ✔ a3fa5ac7ea3d Pull complete                                                                                                                                                                           16.2s
 ✔ es02 Pulled                                                                                                                                                                                            19.2s
 ✔ kibana 12 layers [⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿]      0B/0B      Pulled                                                                                                                                                 35.2s
   ✔ a297b0532e1f Pull complete                                                                                                                                                                            0.7s
   ✔ 2fc7c1dcd340 Pull complete                                                                                                                                                                            0.8s
   ✔ 45499b53179a Pull complete                                                                                                                                                                            0.8s
   ✔ 1244a55fe4a1 Pull complete                                                                                                                                                                            1.0s
   ✔ b029c938ed5e Pull complete                                                                                                                                                                            1.1s
   ✔ b1c3fe59723a Pull complete                                                                                                                                                                           31.4s
   ✔ 196aaf0e5615 Pull complete                                                                                                                                                                           31.5s
   ✔ d5a0d11fa9ac Pull complete                                                                                                                                                                           31.5s
   ✔ ef8c18a35f06 Pull complete                                                                                                                                                                           31.5s
   ✔ 79fddc393150 Pull complete                                                                                                                                                                           31.5s
   ✔ 79b85d016c68 Pull complete                                                                                                                                                                           31.6s
   ✔ 337c99c19bfb Pull complete                                                                                                                                                                           31.6s
 ✔ setup Pulled                                                                                                                                                                                           19.2s
[+] Running 6/6
 ✔ Network elk_default     Created                                                                                                                                                                         0.1s
 ✔ Container elk-setup-1   Healthy                                                                                                                                                                         7.0s
 ✔ Container elk-es01-1    Healthy                                                                                                                                                                        64.9s
 ✔ Container elk-es02-1    Healthy                                                                                                                                                                        55.3s
 ✔ Container elk-es03-1    Healthy                                                                                                                                                                        46.3s
 ✔ Container elk-kibana-1  Started                                                                                                                                                                        65.6s
root@dos:/opt/elk# docker-compose ps
NAME                IMAGE               COMMAND             SERVICE             CREATED             STATUS              PORTS
root@dos:/opt/elk# docker ps -a | grep docker.elastic.co
5e8b7f3b850a   docker.elastic.co/kibana/kibana:8.9.1                 "/bin/tini -- /usr/l…"   9 minutes ago    Exited (1) About a minute ago                                                                                                       elk-kibana-1
4db677726300   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   9 minutes ago    Exited (137) 57 seconds ago                                                                                                         elk-es03-1
19657df10c7b   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   9 minutes ago    Exited (137) 7 minutes ago                                                                                                          elk-es02-1
7e022a4dab84   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   9 minutes ago    Exited (137) 7 minutes ago                                                                                                          elk-es01-1
5a5c07cc1e3f   docker.elastic.co/elasticsearch/elasticsearch:8.9.1   "/bin/tini -- /usr/l…"   9 minutes ago    Exited (0) 8 minutes ago                                                                                                            elk-setup-1
root@dos:/opt/elk#

My first thought is do you have enough memory on your server / laptop?

And / Or Have you allocated enough memory / CPU to docker? If not the nodes will not all start up the it will fail.

I would clean up / prune everything and make sure you have enough resources.

It looks like you are trying to set JVM Heap to 512m generally I recommend 1Gb minimum...

Can you please share the .env?

{"log":"{\"@timestamp\":\"2023-09-04T14:21:01.189Z\", \"log.level\": \"INFO\", \"message\":\"added {{es01}{mAZ2Xh0rRaa4EwhT0GHR_w}{zM47PqAiS2OdlDsKn41EnQ}{es01}{172.30.0.3}{172.30.0.3:9300}{cdfhilmrstw}{8.9.1}}, term: 2, version: 35, reason: ApplyCommitRequest{term=2, version=35, sourceNode={es02}{0QnyKcWaRgyJNiQvtTKwHA}{dbbSVXbHTjKBo9DfBbXLyA}{es02}{172.30.0.4}{172.30.0.4:9300}{cdfhilmrstw}{8.9.1}{ml.allocated_processors_double=8.0, ml.max_jvm_size=536870912, ml.allocated_processors=8, ml.machine_memory=1073741824, xpack.installed=true}}\", \"ecs.version\": \"1.2.0\",\"service.name\":\"ES_ECS\",\"event.dataset\":\"elasticsearch.server\",\"process.thread.name\":\"elasticsearch[es03][clusterApplierService#updateTask][T#1]\",\"log.logger\":\"org.elasticsearch.cluster.service.ClusterApplierService\",\"elasticsearch.cluster.uuid\":\"JeWz8oRTSRqjnntRmtNPQg\",\"elasticsearch.node.id\":\"ROD5xYFqS9iiqaMN9imr5g\",\"elasticsearch.node.name\":\"es03\",\"elasticsearch.cluster.name\":\"kalei\"}\n","stream":"stdout","time":"2023-09-04T14:21:01.190185157Z"}
{"log":"\n","stream":"stderr","time":"2023-09-04T14:21:41.691955437Z"
.....
{"log":"ERROR: Elasticsearch exited unexpectedly\n","stream":"stderr","time":"2023-09-04T14:21:41.692353332Z"}

That unexpectedly quit looks like a OOM perhaps...

Give the Docker ENV more resource that is what I would try first.

set JVM to 1GB and (which means each Container will take 2GB) then you have Kibana as well.

I think you may have a resources issues...

I did prune all already, and i have plenty of resources (running a e2-standard-8@GCP w/ 8 vCPU, 32GB memory), i'm monitoring stats via docker stats and after i start it all, i have about 14gb left, so it's not an issue with resources for sure...

root@dos:/opt/elk# cat .env | sed 's/_PASSWORD=.*/_PASSWORD=/'
# Password for the 'elastic' user (at least 6 characters)
ELASTIC_PASSWORD=

# Password for the 'kibana_system' user (at least 6 characters)
KIBANA_PASSWORD=

# Version of Elastic products
STACK_VERSION=8.9.1

# Set the cluster name
CLUSTER_NAME=kalei

# Set to 'basic' or 'trial' to automatically start the 30-day trial
LICENSE=basic
#LICENSE=trial

# Port to expose Elasticsearch HTTP API to the host
ES_PORT=9200
#ES_PORT=127.0.0.1:9200

# Port to expose Kibana to the host
KIBANA_PORT=5601
#KIBANA_PORT=80

# Increase or decrease based on the available host memory (in bytes)
MEM_LIMIT=1073741824

# Project namespace (defaults to the current folder name if not set)
#COMPOSE_PROJECT_NAME=myproject
root@dos:/opt/elk#

Ok so did you restart with the 1GB?

The previous runs looked like 512mb

All I can say when I see "Unexpectedly Exit" like that error message is that more often than not related to OOM out of memory/resources.... I guess it could be a network thing, because they can not connect to each other ... but that is probably because they are failing.

BTW the reason you need to prune all / get rid of the mounts is because if the setup sees the files / certs etc it will not re-create them ... so be sure you are sure you are clearing the mounts.

You can do this.... try single node... (you can trim your own down or try this)

Here is basically a single node
Clean up and try.

version: "2.2"

services:
  setup:
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
    user: "0"
    command: >
      bash -c '
        if [ x${ELASTIC_PASSWORD} == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x${KIBANA_PASSWORD} == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
          exit 1;
        fi;
        if [ ! -f certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
        fi;
        if [ ! -f certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \
          "instances:\n"\
          "  - name: es01\n"\
          "    dns:\n"\
          "      - es01\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          > config/certs/instances.yml;
          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
          unzip config/certs/certs.zip -d config/certs;
        fi;
        echo "Setting file permissions"
        chown -R root:root config/certs;
        find . -type d -exec chmod 750 \{\} \;;
        find . -type f -exec chmod 640 \{\} \;;
        echo "Waiting for Elasticsearch availability";
        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
        echo "All done!";
      '
    healthcheck:
      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
      interval: 1s
      timeout: 5s
      retries: 120

  es01:
    depends_on:
      setup:
        condition: service_healthy
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - esdata01:/usr/share/elasticsearch/data
    ports:
      - ${ES_PORT}:9200
    environment:
      - node.name=es01
      - cluster.name=${CLUSTER_NAME}
      - cluster.initial_master_nodes=es01
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es01/es01.key
      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es01/es01.key
      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${LICENSE}
    mem_limit: ${MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  kibana:
    depends_on:
      es01:
        condition: service_healthy
    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
    volumes:
      - certs:/usr/share/kibana/config/certs
      - kibanadata:/usr/share/kibana/data
    ports:
      - ${KIBANA_PORT}:5601
    environment:
      - SERVERNAME=kibana
      - ELASTICSEARCH_HOSTS=https://es01:9200
      - ELASTICSEARCH_USERNAME=kibana_system
      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
    mem_limit: ${MEM_LIMIT}
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

volumes:
  certs:
    driver: local
  esdata01:
    driver: local
  kibanadata:
    driver: local

``1`

I did not touch any settings other what's described under Prepare the environment (.env file), i believe as well: because nodes are not able to connect one another, eventually each node "gives up", by producing "Unexpectedly Exit" message.

I'm okay to run docker system prune --all in my environment to prune everything, i ran it and include it with one of my comments before).

my original question is for specifically for:

Start a multi-node cluster with Docker Compose

.. assuming the elasticsearch node is not required to connect to another node to form cluster (due to cluster has single node only), the cluster should become green eventually ..

in my case, i'm required to start a multiple nodes elasticsearch cluster.

and to accomplish that, I started e2-standard-8 (vCPU: 8 & memory: 32 GB) VM and installed Docker and besides Set vm.max_map_count to at least 262144 (Linux) and few other minor changes that I described in my initial question, I did not change anything else, and yet unable to start Elasticsearch with Docker and asking for help to troubleshoot please?) with cherry on the top))

Thank you in advance!

Hi @alexus

Thanks for the details....

Perhaps let me explain why I am asking some these questions / giving thoughts on what to try.

In order to debug the issues and help solve I / we (who have solved many of these issues) need to ask some questions and have the user try some different things because often the reason for the error is not obvious as in this case ... if we just try the same things / step we will most likely get the same result.

Yes thank you and I saw that you ran that, and I had an experience with another user where that command for whatever reason was NOT cleaning up the mounts (and thus the old mounts were still there) and so no matter how many times he ran prune and restarted the docker it did not work until he made sure the mounts were removed. This is why I asked this specifically

I understand, but if you can not even start a single node cluster, you will never be able to start a multi-node cluster, so I suggested starting a single-node cluster to observe, debug, and collect additional data points.

If you can start a Single Node but not a Multi-Node that may point to a FW, Network, docker network etc issue
If you can not start even a single node we might try some other things....

Yes, I understand ... the only way we are going to solve this is by trying different things until we can figure out what is blocking the cluster from forming.

asking for help to troubleshoot please?

That is exactly what I am trying to do ... help... but to help ... especially through a text topic... There needs to be some Debug / Troubleshooting Cycles ... I can not guess and Magically produce an answer ... I wish I could :slight_smile:

You could also try your docker compose on another host or your laptop?

Let me know what you would like to do next...

I really do appreciate all your effort and time you're putting into this, thank you :wink:

and because docker system prune --all is so unreliable, I did start my initial question with manually removing of respective volumes before starting it and thus guaranteeing fresh install.

re: single-node elasticsearch cluster

I took bitnami/elasticsearch image and i was able to get it up and running.. is that okay? or do you want me to get up the official image instead?

I also took very same official docker-compose.yml file and ran it on my laptop via docker-desktop and everything works as it should, no issues there.

To make it easier, maybe I can hop on your Slack :wink:

Thanks again!

No apologies that is not how it works for me.

Not sure if running the bitnami image adds to the discussion / or helps debug

So I will go back to you should try to run Single Node Elasticsearch with Kibana using our Compose edited down by you to a single node / or the compose I provided.

So the full compose works on your laptop: Good / Interesting.

There is something basic going on...

hmm

root@dos:/opt/elk1# docker-compose up -d
[+] Running 7/7
 ✔ Network elk1_default      Created                                                                                                                                                                       0.1s
 ✔ Volume "elk1_certs"       Created                                                                                                                                                                       0.0s
 ✔ Volume "elk1_esdata01"    Created                                                                                                                                                                       0.0s
 ✔ Volume "elk1_kibanadata"  Created                                                                                                                                                                       0.0s
 ✔ Container elk1-setup-1    Healthy                                                                                                                                                                       5.0s
 ✘ Container elk1-es01-1     Error                                                                                                                                                                        39.3s
 ✔ Container elk1-kibana-1   Created                                                                                                                                                                       0.0s
dependency failed to start: container elk1-es01-1 exited (137)
root@dos:/opt/elk1# docker logs elk1-es01-1
Created elasticsearch keystore in /usr/share/elasticsearch/config/elasticsearch.keystore
{"@timestamp":"2023-09-08T00:07:09.726Z", "log.level": "INFO", "message":"version[8.8.1], pid[173], build[docker/f8edfccba429b6477927a7c1ce1bc6729521305e/2023-06-05T21:32:25.188464208Z], OS[Linux/6.1.0-11-cloud-amd64/amd64], JVM[Oracle Corporation/OpenJDK 64-Bit Server VM/20.0.1/20.0.1+9-29]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:09.731Z", "log.level": "INFO", "message":"JVM home [/usr/share/elasticsearch/jdk], using bundled JDK [true]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:09.732Z", "log.level": "INFO", "message":"JVM arguments [-Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -Djava.security.manager=allow, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Dlog4j2.formatMsgNoLookups=true, -Djava.locale.providers=SPI,COMPAT, --add-opens=java.base/java.io=org.elasticsearch.preallocate, -Des.cgroups.hierarchy.override=/, -XX:+UseG1GC, -Djava.io.tmpdir=/tmp/elasticsearch-2014936584690180790, -XX:+HeapDumpOnOutOfMemoryError, -XX:+ExitOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,level,pid,tags:filecount=32,filesize=64m, -Xms512m, -Xmx512m, -XX:MaxDirectMemorySize=268435456, -XX:G1HeapRegionSize=4m, -XX:InitiatingHeapOccupancyPercent=30, -XX:G1ReservePercent=15, -Des.distribution.type=docker, --module-path=/usr/share/elasticsearch/lib, --add-modules=jdk.net, --add-modules=org.elasticsearch.preallocate, -Djdk.module.main=org.elasticsearch.server]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:13.341Z", "log.level": "WARN", "message":"SLF4J: No SLF4J providers were found.", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"stderr","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:13.342Z", "log.level": "WARN", "message":"SLF4J: Defaulting to no-operation (NOP) logger implementation", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"stderr","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:13.342Z", "log.level": "WARN", "message":"SLF4J: See https://www.slf4j.org/codes.html#noProviders for further details.", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"stderr","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:13.343Z", "log.level": "WARN", "message":"SLF4J: Class path contains SLF4J bindings targeting slf4j-api versions 1.7.x or earlier.", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"stderr","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:13.344Z", "log.level": "WARN", "message":"SLF4J: Ignoring binding found at [jar:file:///usr/share/elasticsearch/modules/repository-azure/log4j-slf4j-impl-2.19.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"stderr","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:13.345Z", "log.level": "WARN", "message":"SLF4J: See https://www.slf4j.org/codes.html#ignoredBindings for an explanation.", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"stderr","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.857Z", "log.level": "INFO", "message":"loaded module [repository-url]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.858Z", "log.level": "INFO", "message":"loaded module [dlm]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.858Z", "log.level": "INFO", "message":"loaded module [x-pack-core]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.858Z", "log.level": "INFO", "message":"loaded module [x-pack-redact]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.859Z", "log.level": "INFO", "message":"loaded module [ingest-user-agent]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.859Z", "log.level": "INFO", "message":"loaded module [x-pack-async-search]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.859Z", "log.level": "INFO", "message":"loaded module [x-pack-monitoring]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.859Z", "log.level": "INFO", "message":"loaded module [repository-s3]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.859Z", "log.level": "INFO", "message":"loaded module [x-pack-analytics]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.860Z", "log.level": "INFO", "message":"loaded module [x-pack-ent-search]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.860Z", "log.level": "INFO", "message":"loaded module [x-pack-autoscaling]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.861Z", "log.level": "INFO", "message":"loaded module [lang-painless]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.862Z", "log.level": "INFO", "message":"loaded module [x-pack-ml]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.862Z", "log.level": "INFO", "message":"loaded module [legacy-geo]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.862Z", "log.level": "INFO", "message":"loaded module [lang-mustache]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.862Z", "log.level": "INFO", "message":"loaded module [x-pack-ql]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.862Z", "log.level": "INFO", "message":"loaded module [rank-rrf]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.862Z", "log.level": "INFO", "message":"loaded module [analysis-common]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.862Z", "log.level": "INFO", "message":"loaded module [transport-netty4]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.863Z", "log.level": "INFO", "message":"loaded module [aggregations]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.863Z", "log.level": "INFO", "message":"loaded module [ingest-common]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.863Z", "log.level": "INFO", "message":"loaded module [x-pack-identity-provider]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.863Z", "log.level": "INFO", "message":"loaded module [frozen-indices]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.863Z", "log.level": "INFO", "message":"loaded module [x-pack-shutdown]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.863Z", "log.level": "INFO", "message":"loaded module [x-pack-text-structure]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.863Z", "log.level": "INFO", "message":"loaded module [snapshot-repo-test-kit]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.863Z", "log.level": "INFO", "message":"loaded module [ml-package-loader]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.864Z", "log.level": "INFO", "message":"loaded module [kibana]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.864Z", "log.level": "INFO", "message":"loaded module [constant-keyword]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.865Z", "log.level": "INFO", "message":"loaded module [x-pack-logstash]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.865Z", "log.level": "INFO", "message":"loaded module [x-pack-ccr]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.866Z", "log.level": "INFO", "message":"loaded module [x-pack-graph]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.866Z", "log.level": "INFO", "message":"loaded module [parent-join]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.866Z", "log.level": "INFO", "message":"loaded module [x-pack-enrich]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.866Z", "log.level": "INFO", "message":"loaded module [repositories-metering-api]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.866Z", "log.level": "INFO", "message":"loaded module [transform]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.866Z", "log.level": "INFO", "message":"loaded module [repository-azure]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.866Z", "log.level": "INFO", "message":"loaded module [repository-gcs]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.866Z", "log.level": "INFO", "message":"loaded module [spatial]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.866Z", "log.level": "INFO", "message":"loaded module [mapper-version]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.867Z", "log.level": "INFO", "message":"loaded module [apm]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.867Z", "log.level": "INFO", "message":"loaded module [mapper-extras]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.867Z", "log.level": "INFO", "message":"loaded module [x-pack-aggregate-metric]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.867Z", "log.level": "INFO", "message":"loaded module [x-pack-rollup]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.867Z", "log.level": "INFO", "message":"loaded module [percolator]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.867Z", "log.level": "INFO", "message":"loaded module [data-streams]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.867Z", "log.level": "INFO", "message":"loaded module [x-pack-stack]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.867Z", "log.level": "INFO", "message":"loaded module [reindex]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.868Z", "log.level": "INFO", "message":"loaded module [rank-eval]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.868Z", "log.level": "INFO", "message":"loaded module [x-pack-security]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.868Z", "log.level": "INFO", "message":"loaded module [blob-cache]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.869Z", "log.level": "INFO", "message":"loaded module [searchable-snapshots]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.869Z", "log.level": "INFO", "message":"loaded module [snapshot-based-recoveries]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.869Z", "log.level": "INFO", "message":"loaded module [x-pack-watcher]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.870Z", "log.level": "INFO", "message":"loaded module [old-lucene-versions]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.870Z", "log.level": "INFO", "message":"loaded module [x-pack-ilm]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.870Z", "log.level": "INFO", "message":"loaded module [x-pack-voting-only-node]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.870Z", "log.level": "INFO", "message":"loaded module [x-pack-deprecation]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.870Z", "log.level": "INFO", "message":"loaded module [x-pack-fleet]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.871Z", "log.level": "INFO", "message":"loaded module [x-pack-profiling]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.871Z", "log.level": "INFO", "message":"loaded module [ingest-geoip]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.871Z", "log.level": "INFO", "message":"loaded module [x-pack-write-load-forecaster]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.871Z", "log.level": "INFO", "message":"loaded module [search-business-rules]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.871Z", "log.level": "INFO", "message":"loaded module [ingest-attachment]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.871Z", "log.level": "INFO", "message":"loaded module [wildcard]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.871Z", "log.level": "INFO", "message":"loaded module [x-pack-sql]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.871Z", "log.level": "INFO", "message":"loaded module [unsigned-long]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.871Z", "log.level": "INFO", "message":"loaded module [x-pack-async]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.871Z", "log.level": "INFO", "message":"loaded module [runtime-fields-common]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.872Z", "log.level": "INFO", "message":"loaded module [vector-tile]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.872Z", "log.level": "INFO", "message":"loaded module [lang-expression]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:15.872Z", "log.level": "INFO", "message":"loaded module [x-pack-eql]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.plugins.PluginsService","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:21.848Z", "log.level": "WARN", "message":"Sep 08, 2023 12:07:21 AM org.apache.lucene.store.MemorySegmentIndexInputProvider <init>", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"stderr","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:21.864Z", "log.level": "INFO", "message":"using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/sdb)]], net usable_space [79.8gb], net total_space [97.8gb], types [ext4]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.env.NodeEnvironment","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:21.865Z", "log.level": "INFO", "message":"heap size [512mb], compressed ordinary object pointers [true]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.env.NodeEnvironment","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:21.883Z", "log.level": "INFO", "message":"node name [es01], node ID [-8DEj-vnTJ6wrvWxfSWxQg], cluster name [kalei], roles [ml, data_frozen, ingest, data_cold, data, remote_cluster_client, master, data_warm, data_content, transform, data_hot]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.node.Node","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:27.561Z", "log.level": "INFO", "message":"[controller/199] [Main.cc@123] controller (64 bit): Version 8.8.1 (Build 4658a65098c570) Copyright (c) 2023 Elasticsearch BV", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"ml-cpp-log-tail-thread","log.logger":"org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:27.940Z", "log.level": "INFO", "message":"Security is enabled", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.xpack.security.Security","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:29.404Z", "log.level": "INFO", "message":"parsed [0] roles from file [/usr/share/elasticsearch/config/roles.yml]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.xpack.security.authz.store.FileRolesStore","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}
{"@timestamp":"2023-09-08T00:07:30.902Z", "log.level": "INFO", "message":"Profiling is enabled", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"main","log.logger":"org.elasticsearch.xpack.profiler.ProfilingPlugin","elasticsearch.node.name":"es01","elasticsearch.cluster.name":"kalei"}

ERROR: Elasticsearch exited unexpectedly
root@dos:/opt/elk1#