[2023-06-30T13:43:27,396][ERROR][o.e.x.m.c.c.ClusterStatsCollector] [xxx-es-master-2.xxx.com] collector [cluster_stats] failed to collect data
org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:661) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:384) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:694) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:467) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:292) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:226) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:424) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:184) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:668) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:489) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:283) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:103) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:76) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:324) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:212) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:91) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:173) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$3(SecurityActionFilter.java:160) ~[?:?]
at org.elasticsearch.action.ActionListener$DelegatingFailureActionListener.onResponse(ActionListener.java:217) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:101) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:95) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture.lambda$done$0(ListenableFuture.java:87) ~[elasticsearch-7.13.4.jar:7.13.4]
at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?]
at org.elasticsearch.common.util.concurrent.ListenableFuture.done(ListenableFuture.java:87) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.BaseFuture.set(BaseFuture.java:133) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture.onResponse(ListenableFuture.java:116) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.StepListener.innerOnResponse(StepListener.java:53) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.xpack.security.authz.interceptor.ResizeRequestInterceptor.intercept(ResizeRequestInterceptor.java:87) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$runRequestInterceptors$14(AuthorizationService.java:380) ~[?:?]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:101) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:95) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture.lambda$done$0(ListenableFuture.java:87) ~[elasticsearch-7.13.4.jar:7.13.4]
at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?]
at org.elasticsearch.common.util.concurrent.ListenableFuture.done(ListenableFuture.java:87) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.BaseFuture.set(BaseFuture.java:133) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture.onResponse(ListenableFuture.java:116) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.StepListener.innerOnResponse(StepListener.java:53) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.xpack.security.authz.interceptor.FieldAndDocumentLevelSecurityRequestInterceptor.intercept(FieldAndDocumentLevelSecurityRequestInterceptor.java:77) ~[?:?]
at org.elasticsearch.xpack.security.authz.interceptor.UpdateRequestInterceptor.intercept(UpdateRequestInterceptor.java:24) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$runRequestInterceptors$14(AuthorizationService.java:380) ~[?:?]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:101) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:95) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture.lambda$done$0(ListenableFuture.java:87) ~[elasticsearch-7.13.4.jar:7.13.4]
at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?]
at org.elasticsearch.common.util.concurrent.ListenableFuture.done(ListenableFuture.java:87) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.BaseFuture.set(BaseFuture.java:133) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture.onResponse(ListenableFuture.java:116) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.StepListener.innerOnResponse(StepListener.java:53) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.xpack.security.authz.interceptor.IndicesAliasesRequestInterceptor.intercept(IndicesAliasesRequestInterceptor.java:107) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$runRequestInterceptors$14(AuthorizationService.java:380) ~[?:?]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:101) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:95) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture.lambda$done$0(ListenableFuture.java:87) ~[elasticsearch-7.13.4.jar:7.13.4]
at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?]
at org.elasticsearch.common.util.concurrent.ListenableFuture.done(ListenableFuture.java:87) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.BaseFuture.set(BaseFuture.java:133) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture.onResponse(ListenableFuture.java:116) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.StepListener.innerOnResponse(StepListener.java:53) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.xpack.security.authz.interceptor.BulkShardRequestInterceptor.intercept(BulkShardRequestInterceptor.java:77) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$runRequestInterceptors$14(AuthorizationService.java:380) ~[?:?]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:101) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:215) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:95) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture.lambda$done$0(ListenableFuture.java:87) ~[elasticsearch-7.13.4.jar:7.13.4]
at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?]
at org.elasticsearch.common.util.concurrent.ListenableFuture.done(ListenableFuture.java:87) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.BaseFuture.set(BaseFuture.java:133) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ListenableFuture.onResponse(ListenableFuture.java:116) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.StepListener.innerOnResponse(StepListener.java:53) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.NotifyOnceListener.onResponse(NotifyOnceListener.java:29) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.xpack.security.authz.interceptor.FieldAndDocumentLevelSecurityRequestInterceptor.intercept(FieldAndDocumentLevelSecurityRequestInterceptor.java:77) ~[?:?]
at org.elasticsearch.xpack.security.authz.interceptor.SearchRequestInterceptor.intercept(SearchRequestInterceptor.java:22) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService.runRequestInterceptors(AuthorizationService.java:386) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService.handleIndexActionAuthorizationResult(AuthorizationService.java:363) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$9(AuthorizationService.java:303) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:688) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:663) ~[?:?]
at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.xpack.security.authz.RBACEngine.buildIndicesAccessControl(RBACEngine.java:556) ~[?:?]
at org.elasticsearch.xpack.security.authz.RBACEngine.lambda$authorizeIndexAction$3(RBACEngine.java:320) ~[?:?]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.lambda$getAsync$0(AuthorizationService.java:726) ~[?:?]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.xpack.security.authz.AuthorizationService.resolveIndexNames(AuthorizationService.java:600) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$6(AuthorizationService.java:291) ~[?:?]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.lambda$getAsync$0(AuthorizationService.java:726) ~[?:?]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.xpack.security.authz.RBACEngine.loadAuthorizedIndices(RBACEngine.java:367) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$5(AuthorizationService.java:287) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.getAsync(AuthorizationService.java:724) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$8(AuthorizationService.java:290) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.getAsync(AuthorizationService.java:724) ~[?:?]
at org.elasticsearch.xpack.security.authz.RBACEngine.authorizeIndexAction(RBACEngine.java:311) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:301) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:266) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:230) ~[?:?]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:31) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:127) ~[?:?]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:220) ~[?:?]
at org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:133) ~[?:?]
at org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:121) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:232) ~[?:?]
at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.authorizeRequest(SecurityActionFilter.java:178) ~[?:?]
at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$4(SecurityActionFilter.java:159) ~[?:?]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:134) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$0(AuthenticationService.java:336) ~[?:?]
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$8(AuthenticationService.java:414) ~[?:?]
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:425) ~[?:?]
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:333) ~[?:?]
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:274) ~[?:?]
at org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:167) ~[?:?]
at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.applyInternal(SecurityActionFilter.java:154) ~[?:?]
at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$apply$1(SecurityActionFilter.java:102) ~[?:?]
at org.elasticsearch.xpack.core.security.SecurityContext.executeAsUser(SecurityContext.java:139) ~[?:?]
at org.elasticsearch.xpack.security.authz.AuthorizationUtils.switchUserBasedOnActionOriginAndExecute(AuthorizationUtils.java:133) ~[?:?]
at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.apply(SecurityActionFilter.java:101) ~[?:?]
at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:171) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:149) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:77) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:86) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:66) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:402) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.client.support.AbstractClient.search(AbstractClient.java:539) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin(ClientHelper.java:119) ~[?:?]
at org.elasticsearch.xpack.transform.TransformFeatureSet.usage(TransformFeatureSet.java:190) ~[?:?]
at org.elasticsearch.xpack.core.action.TransportXPackUsageAction.lambda$masterOperation$2(TransportXPackUsageAction.java:55) ~[?:?]
at org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:118) ~[?:?]
at org.elasticsearch.action.ActionRunnable.lambda$supply$0(ActionRunnable.java:47) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:62) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) ~[elasticsearch-7.13.4.jar:7.13.4]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) ~[elasticsearch-7.13.4.jar:7.13.4]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]
at java.lang.Thread.run(Thread.java:831) [?:?]
Caused by: org.elasticsearch.action.NoShardAvailableActionException
at org.elasticsearch.action.sea
If you want someone to help i believe you will need to provide more details. What is the size and configuration of your cluster? How many master eligible nodes do you have?
3 Master, 3 coordinating, 30 data
Please provide a fuller description of the problem, not just a title and a stack trace. Also provide the full logs from around the time of the issue.
ELK cluster issue failed after data node restart
Tried restarting all nodes, after restart the index starts to restore but during restoring there is an error message:
.text-only,.text-card-text{white-space: pre;}.rich-text-paragraph{min-height: 15px;}failed to join {[xxx-es-master-3.xxx.com](http://xxx-es-master-3.xxx.com/)}{pXK59V3_TECJt3i0QGAmKA}{o0NtTUopT1-voGFC_lwl9Q}{[10.xxx.xx.xx](http://10.xxx.xx.xx/)}{[10.xxx.xx.xx:9300](http://10.xxx.xx.xx:9300/)}{m}{zone=PH-147, xpack.installed=true, transform.node=false} with JoinRequest{sourceNode={[xxx-es-master-1.xxx.com](http://xxx-es-master-1.xxx.com/)}{Q38X-pDmS3uPGVGVRpbTLA}{VDUj0eXISIGnHnBg0VUFQA}{[10.xxx.xx.xx](http://10.xxx.xx.xx/)}{[10.xxx.xx.xx:9300](http://10.xxx.xx.xx:9300/)}{m}{xpack.installed=true, zone=PH-147, transform.node=false}, minimumTerm=990664, optionalJoin=Optional.empty} org.elasticsearch.transport.RemoteTransportException: [[xx-es-master-3.xxx.com](http://xxx-es-master-3.xxx.com/)][[10.xxx.xx.xx:9300](http://10.xxx.xx.xx:9300/)][internal:cluster/coordination/join] Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {[xxx-es-master-1.xxx.com](http://xxx-es-master-1.xxx.com/)}{Q38X-pDmS3uPGVGVRpbTLA}
after this error we restarted one of the master and then we received the error I posted above. Hope these information helps. Thank you.
Also showing kibana error
.text-only,.text-card-text{white-space: pre;}.rich-text-paragraph{min-height: 15px;}{"type":"log","@timestamp":"2023-06-30T15:57:06+08:00","tags":["info","savedobjects-service"],"pid":30220,"message":"[.kibana-147-105] OUTDATED_DOCUMENTS_SEARCH -> OUTDATED_DOCUMENTS_SEARCH. took: 64076ms."}
This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.