Getting below errors
{"@timestamp":"2025-08-13T08:41:40.028Z", "log.level": "INFO", "message":"\[controller/5551\] \[Main.cc@176\] ML controller exiting", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"ml-cpp-log-tail-thread","log.logger":"org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.030Z", "log.level": "INFO", "message":"Native controller process has stopped - no new native processes can be started", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"ml-cpp-log-tail-thread","log.logger":"org.elasticsearch.xpack.ml.process.NativeController","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.030Z", "log.level": "INFO", "message":"adding component template \[traces-apm-fallback@ilm\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.034Z", "log.level": "INFO", "message":"adding component template \[metrics-apm.transaction@mappings\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.036Z", "log.level": "INFO", "message":"adding component template \[logs-apm@settings\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.038Z", "log.level": "INFO", "message":"adding component template \[metrics-apm.service_transaction.1m-fallback@ilm\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.040Z", "log.level": "INFO", "message":"adding component template \[metrics-apm.transaction.60m-fallback@ilm\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.041Z", "log.level": "INFO", "message":"adding component template \[metrics-apm.transaction.1m-fallback@ilm\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.042Z", "log.level": "INFO", "message":"adding component template \[metrics-apm.service_summary.10m-fallback@ilm\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.069Z", "log.level": "INFO", "message":"adding component template \[logs-apm.error@mappings\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.072Z", "log.level": "INFO", "message":"adding component template \[apm@mappings\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.074Z", "log.level": "INFO", "message":"adding component template \[logs-apm.error-fallback@ilm\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.297Z", "log.level": "INFO", "message":"transport connection to \[{virtlog2.srv.gpe}{8Dz9kvnyS02AoxA0UNqZcA}{xlfZaMKtTMC-Q76UkfmX7Q}{virtlog2.srv.gpe}{192.168.121.190}{192.168.121.190:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}\] closed by remote; if unexpected, see \[https://www.elastic.co/guide/en/elasticsearch/reference/8.18/troubleshooting-unstable-cluster.html#troubleshooting-unstable-cluster-network\\\\\\\] for troubleshooting guidance", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[transport_worker\]\[T#2\]","log.logger":"org.elasticsearch.transport.ClusterConnectionManager","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.327Z", "log.level": "WARN", "message":"failed to connect to {virtlog2.srv.gpe}{8Dz9kvnyS02AoxA0UNqZcA}{xlfZaMKtTMC-Q76UkfmX7Q}{virtlog2.srv.gpe}{192.168.121.190}{192.168.121.190:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106227712, xpack.installed=true, transform.config_version=10.0.0} (tried \[1\] times)", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[generic\]\[T#13\]","log.logger":"org.elasticsearch.cluster.NodeConnectionsService","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.transport.ConnectTransportException","error.message":"\[virtlog2.srv.gpe\]\[192.168.121.190:9300\] connect_exception","error.stack_trace":"org.elasticsearch.transport.ConnectTransportException: \[virtlog2.srv.gpe\]\[192.168.121.190:9300\] connect_exception\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:1139)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.action.support.SubscribableListener$FailureResult.complete(SubscribableListener.java:399)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.action.support.SubscribableListener.tryComplete(SubscribableListener.java:311)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.action.support.SubscribableListener.addListener(SubscribableListener.java:210)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.action.support.SubscribableListener.addListener(SubscribableListener.java:175)\\n\\tat org.elasticsearch.transport.netty4@8.18.4/org.elasticsearch.transport.netty4.Netty4TcpChannel.addConnectListener(Netty4TcpChannel.java:95)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.transport.TcpTransport.initiateConnection(TcpTransport.java:405)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.transport.TcpTransport.lambda$openConnection$2(TcpTransport.java:366)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.action.ActionListener.run(ActionListener.java:454)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.transport.TcpTransport.openConnection(TcpTransport.java:353)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.transport.ClusterConnectionManager.internalOpenConnection(ClusterConnectionManager.java:372)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.transport.ClusterConnectionManager.connectToNodeOrRetry(ClusterConnectionManager.java:214)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.transport.ClusterConnectionManager.connectToNode(ClusterConnectionManager.java:113)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:514)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.transport.TransportService.connectToNode(TransportService.java:494)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.NodeConnectionsService$ConnectionTarget.doConnect(NodeConnectionsService.java:289)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.NodeConnectionsService$ConnectionTarget.lambda$connect$0(NodeConnectionsService.java:236)\\n\\tat java.base/java.util.ArrayList.forEach(ArrayList.java:1604)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.NodeConnectionsService.connectToNodes(NodeConnectionsService.java:125)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.service.ClusterApplierService.connectToNodesAsync(ClusterApplierService.java:526)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.service.ClusterApplierService.connectToNodesAndWait(ClusterApplierService.java:516)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:490)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:432)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:157)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:977)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:218)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:184)\\n\\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)\\n\\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)\\n\\tat java.base/java.lang.Thread.run(Thread.java:1447)\\nCaused by: org.elasticsearch.common.util.concurrent.UncategorizedExecutionException: Failed execution\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.action.support.SubscribableListener.wrapAsExecutionException(SubscribableListener.java:292)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.common.util.concurrent.ListenableFuture.wrapException(ListenableFuture.java:39)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.common.util.concurrent.ListenableFuture.wrapException(ListenableFuture.java:28)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.action.support.SubscribableListener.onFailure(SubscribableListener.java:255)\\n\\tat org.elasticsearch.transport.netty4@8.18.4/org.elasticsearch.transport.netty4.Netty4Utils.lambda$addListener$2(Netty4Utils.java:237)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)\\n\\tat org.elasticsearch.security@8.18.4/org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport$ClientSslHandlerInitializer.lambda$connect$1(SecurityNetty4Transport.java:347)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)\\n\\tat io.netty.transport@4.1.118.Final/io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)\\n\\tat io.netty.transport@4.1.118.Final/io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)\\n\\tat io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)\\n\\tat io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:697)\\n\\tat io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:660)\\n\\tat io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\\n\\t... 1 more\\nCaused by: java.util.concurrent.ExecutionException: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 192.168.121.190/192.168.121.190:9300\\n\\t... 29 more\\nCaused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 192.168.121.190/192.168.121.190:9300\\nCaused by: java.net.ConnectException: Connection refused\\n\\tat java.base/sun.nio.ch.Net.pollConnect(Native Method)\\n\\tat java.base/sun.nio.ch.Net.pollConnectNow(Net.java:628)\\n\\tat java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:1046)\\n\\tat io.netty.transport@4.1.118.Final/io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)\\n\\tat io.netty.transport@4.1.118.Final/io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)\\n\\tat io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)\\n\\tat io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:697)\\n\\tat io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:660)\\n\\tat io.netty.transport@4.1.118.Final/io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)\\n\\tat io.netty.common@4.1.118.Final/io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\\n\\tat java.base/java.lang.Thread.run(Thread.java:1447)\\n"}
{"@timestamp":"2025-08-13T08:41:40.328Z", "log.level": "INFO", "message":"node-left\[{virtlog2.srv.gpe}{8Dz9kvnyS02AoxA0UNqZcA}{xlfZaMKtTMC-Q76UkfmX7Q}{virtlog2.srv.gpe}{192.168.121.190}{192.168.121.190:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000} reason: disconnected\], term: 2, version: 7, delta: removed {{virtlog2.srv.gpe}{8Dz9kvnyS02AoxA0UNqZcA}{xlfZaMKtTMC-Q76UkfmX7Q}{virtlog2.srv.gpe}{192.168.121.190}{192.168.121.190:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}}", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.cluster.service.MasterService","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.336Z", "log.level": "WARN", "message":"failing \[node-left\[{virtlog2.srv.gpe}{8Dz9kvnyS02AoxA0UNqZcA}{xlfZaMKtTMC-Q76UkfmX7Q}{virtlog2.srv.gpe}{192.168.121.190}{192.168.121.190:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000} reason: disconnected\]\]: failed to commit cluster state version \[7\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.cluster.service.MasterService","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException","error.message":"publication of cluster state version \[7\] in term \[2\] failed \[committed=false\]","error.stack_trace":"org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication of cluster state version \[7\] in term \[2\] failed \[committed=false\]\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$4.onFailure(Coordinator.java:2059)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.action.support.SubscribableListener$FailureResult.complete(SubscribableListener.java:399)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.action.support.SubscribableListener.tryComplete(SubscribableListener.java:311)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.action.support.SubscribableListener.addListener(SubscribableListener.java:210)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1965)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:114)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:165)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.coordination.Publication.start(Publication.java:61)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.coordination.Coordinator.publish(Coordinator.java:1622)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.service.MasterService.publish(MasterService.java:488)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.service.MasterService.publishClusterStateUpdate(MasterService.java:366)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.service.MasterService.lambda$executeAndPublishBatch$2(MasterService.java:305)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.action.ActionListener.run(ActionListener.java:454)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.service.MasterService.executeAndPublishBatch(MasterService.java:287)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.service.MasterService$BatchingTaskQueue$Processor.lambda$run$2(MasterService.java:1691)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.action.ActionListener.run(ActionListener.java:454)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.service.MasterService$BatchingTaskQueue$Processor.run(MasterService.java:1688)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.service.MasterService$5.lambda$doRun$0(MasterService.java:1283)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.action.ActionListener.run(ActionListener.java:454)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.service.MasterService$5.doRun(MasterService.java:1262)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:1044)\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:27)\\n\\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1095)\\n\\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:619)\\n\\tat java.base/java.lang.Thread.run(Thread.java:1447)\\nCaused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: non-failed nodes do not form a quorum\\n\\tat org.elasticsearch.server@8.18.4/org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:163)\\n\\t... 18 more\\n"}
{"@timestamp":"2025-08-13T08:41:40.340Z", "log.level": "INFO", "message":"master node changed {previous \[{virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}\], current [ ]}, term: 2, version: 6, reason: becoming candidate: Publication.onCompletion(false)", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[clusterApplierService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.cluster.service.ClusterApplierService","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.352Z", "log.level":"ERROR", "message":"unexpected failure during \[maybe generate license for cluster\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.license.StartupSelfGeneratedLicenseTask","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.cluster.NotMasterException","error.message":"no longer master","error.stack_trace":"org.elasticsearch.cluster.NotMasterException: no longer master\\n"}
{"@timestamp":"2025-08-13T08:41:40.353Z", "log.level":"ERROR", "message":"unexpected failure during \[maybe generate license for cluster\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.license.StartupSelfGeneratedLicenseTask","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.cluster.NotMasterException","error.message":"no longer master","error.stack_trace":"org.elasticsearch.cluster.NotMasterException: no longer master\\n"}
{"@timestamp":"2025-08-13T08:41:40.357Z", "log.level":"ERROR", "message":"unexpected failure during \[maybe generate license for cluster\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.license.StartupSelfGeneratedLicenseTask","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.cluster.NotMasterException","error.message":"no longer master","error.stack_trace":"org.elasticsearch.cluster.NotMasterException: no longer master\\n"}
{"@timestamp":"2025-08-13T08:41:40.358Z", "log.level":"ERROR", "message":"unexpected failure during \[maybe generate license for cluster\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[masterService#updateTask\]\[T#1\]","log.logger":"org.elasticsearch.license.StartupSelfGeneratedLicenseTask","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.cluster.NotMasterException","error.message":"no longer master","error.stack_trace":"org.elasticsearch.cluster.NotMasterException: no longer master\\n"}
{"@timestamp":"2025-08-13T08:41:40.383Z", "log.level": "INFO", "message":"transport connection to \[{virtlog1.srv.gpe}{fiuHo05nQU-3udVdRF5iZw}{b7OZ7sNIT_iF7pRDGj1n5w}{virtlog1.srv.gpe}{192.168.121.177}{192.168.121.177:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}\] closed by remote; if unexpected, see \[https://www.elastic.co/guide/en/elasticsearch/reference/8.18/troubleshooting-unstable-cluster.html#troubleshooting-unstable-cluster-network\\\\\\\] for troubleshooting guidance", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch\[virtlog3.srv.gpe\]\[transport_worker\]\[T#2\]","log.logger":"org.elasticsearch.transport.ClusterConnectionManager","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment"}
{"@timestamp":"2025-08-13T08:41:40.439Z", "log.level":"ERROR", "message":"error adding index template \[metrics-service_destination.60m@template\] for \[otel\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch-shutdown","log.logger":"org.elasticsearch.xpack.core.template.IndexTemplateRegistry","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.node.NodeClosedException","error.message":"node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}","error.stack_trace":"org.elasticsearch.node.NodeClosedException: node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}\\n"}
{"@timestamp":"2025-08-13T08:41:40.440Z", "log.level":"ERROR", "message":"error adding index template \[.kibana-reporting\] for \[stack\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch-shutdown","log.logger":"org.elasticsearch.xpack.core.template.IndexTemplateRegistry","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.node.NodeClosedException","error.message":"node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}","error.stack_trace":"org.elasticsearch.node.NodeClosedException: node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}\\n"}
{"@timestamp":"2025-08-13T08:41:40.441Z", "log.level":"ERROR", "message":"error adding lifecycle policy \[logs-apm.error_logs-default_policy\] for \[apm\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch-shutdown","log.logger":"org.elasticsearch.xpack.core.template.IndexTemplateRegistry","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.node.NodeClosedException","error.message":"node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}","error.stack_trace":"org.elasticsearch.node.NodeClosedException: node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}\\n"}
{"@timestamp":"2025-08-13T08:41:40.441Z", "log.level":"ERROR", "message":"error adding index template \[logs-apm.error@template\] for \[apm\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch-shutdown","log.logger":"org.elasticsearch.xpack.core.template.IndexTemplateRegistry","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.node.NodeClosedException","error.message":"node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}","error.stack_trace":"org.elasticsearch.node.NodeClosedException: node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}\\n"}
{"@timestamp":"2025-08-13T08:41:40.442Z", "log.level":"ERROR", "message":"error adding index template \[metrics-apm.service_destination.1m@template\] for \[apm\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch-shutdown","log.logger":"org.elasticsearch.xpack.core.template.IndexTemplateRegistry","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.node.NodeClosedException","error.message":"node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}","error.stack_trace":"org.elasticsearch.node.NodeClosedException: node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}\\n"}
{"@timestamp":"2025-08-13T08:41:40.442Z", "log.level":"ERROR", "message":"error adding index template \[.deprecation-indexing-template\] for \[deprecation\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch-shutdown","log.logger":"org.elasticsearch.xpack.core.template.IndexTemplateRegistry","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.node.NodeClosedException","error.message":"node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}","error.stack_trace":"org.elasticsearch.node.NodeClosedException: node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}\\n"}
{"@timestamp":"2025-08-13T08:41:40.443Z", "log.level":"ERROR", "message":"error adding ingest pipeline template \[apm@pipeline\] for \[apm\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch-shutdown","log.logger":"org.elasticsearch.xpack.core.template.IndexTemplateRegistry","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.node.NodeClosedException","error.message":"node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}","error.stack_trace":"org.elasticsearch.node.NodeClosedException: node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}\\n"}
{"@timestamp":"2025-08-13T08:41:40.443Z", "log.level":"ERROR", "message":"error adding lifecycle policy \[metrics-apm.service_summary_10m_metrics-default_policy\] for \[apm\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch-shutdown","log.logger":"org.elasticsearch.xpack.core.template.IndexTemplateRegistry","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.node.NodeClosedException","error.message":"node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}","error.stack_trace":"org.elasticsearch.node.NodeClosedException: node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}\\n"}
{"@timestamp":"2025-08-13T08:41:40.444Z", "log.level":"ERROR", "message":"error adding index template \[elastic-connectors-sync-jobs\] for \[enterprise_search\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch-shutdown","log.logger":"org.elasticsearch.xpack.core.template.IndexTemplateRegistry","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.node.NodeClosedException","error.message":"node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}","error.stack_trace":"org.elasticsearch.node.NodeClosedException: node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}\\n"}
{"@timestamp":"2025-08-13T08:41:40.444Z", "log.level":"ERROR", "message":"error adding ingest pipeline template \[logs@json-message\] for \[stack\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch-shutdown","log.logger":"org.elasticsearch.xpack.core.template.IndexTemplateRegistry","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.node.NodeClosedException","error.message":"node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}","error.stack_trace":"org.elasticsearch.node.NodeClosedException: node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}\\n"}
{"@timestamp":"2025-08-13T08:41:40.445Z", "log.level":"ERROR", "message":"error adding index template \[traces-apm.rum@template\] for \[apm\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch-shutdown","log.logger":"org.elasticsearch.xpack.core.template.IndexTemplateRegistry","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.node.NodeClosedException","error.message":"node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}","error.stack_trace":"org.elasticsearch.node.NodeClosedException: node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}\\n"}
{"@timestamp":"2025-08-13T08:41:40.445Z", "log.level":"ERROR", "message":"error adding lifecycle policy \[metrics-apm.app_metrics-default_policy\] for \[apm\]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch-shutdown","log.logger":"org.elasticsearch.xpack.core.template.IndexTemplateRegistry","elasticsearch.cluster.uuid":"dziZws4KQVyCCGWi2UQqxg","elasticsearch.node.id":"mTgpG6XIQ--zbX-WXPAFRA","elasticsearch.node.name":"virtlog3.srv.gpe","elasticsearch.cluster.name":"logging-segment","error.type":"org.elasticsearch.node.NodeClosedException","error.message":"node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}","error.stack_trace":"org.elasticsearch.node.NodeClosedException: node closed {virtlog3.srv.gpe}{mTgpG6XIQ--zbX-WXPAFRA}{AFctZywfRW6sn9Rl4KI5KQ}{virtlog3.srv.gpe}{192.168.121.159}{192.168.121.159:9300}{cdfhilmrstw}{8.18.4}{7000099-8525000}{ml.config_version=12.0.0, ml.max_jvm_size=1073741824, ml.allocated_processors_double=2.0, ml.allocated_processors=2, ml.machine_memory=4106231808, transform.config_version=10.0.0, xpack.installed=true}\\n"}
and Elasticsearch.yaml file is as follows
cluster.initial_master_nodes:
\- 192.xxx.xxx.38
\- 192.xxx.xxx.90
\- 192.xxx.xxx.54
cluster.name: logging-segment
cluster.routing.allocation.disk.watermark.flood_stage: 90%
cluster.routing.allocation.disk.watermark.high: 85%
cluster.routing.allocation.disk.watermark.low: 80%
cluster.routing.allocation.node_concurrent_recoveries: 5
discovery.seed_hosts:
\- 192.xxx.xxx.38
\- 192.xxx.xxx.90
\- 192.xxx.xxx.54
http.max_content_length: 512mb
http.port: 9200
ingest.geoip.downloader.enabled: false
network.host: 192.xxx.xxx.38
node.name: virtlog1.srv.gpe
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
xpack.security.authc:
anonymous:
authz_exception: true
roles: monitor
username: \_anonymous
xpack.security.authc.realms:
file:
admin_fallback:
order: 1
native:
user_store:
order: 0
xpack.security.enabled: true
xpack.security.enrollment.enabled: false
xpack.security.http.ssl.certificate: /etc/elasticsearch/ssl/elasticsearch.crt
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.key: /etc/elasticsearch/ssl/elasticsearch.key
xpack.security.http.ssl.verification_mode: certificate
xpack.security.transport.ssl.certificate: /etc/elasticsearch/ssl/elasticsearch.crt
xpack.security.transport.ssl.certificate_authorities: /etc/elasticsearch/ssl/GPE-BTS-SST-Root-G1.crt
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.key: /etc/elasticsearch/ssl/elasticsearch.key
xpack.security.transport.ssl.verification_mode: certificate