Hi @warkolm, below is the response when I execute GET sam--000110/_ilm/explain
As per ILM things were working fine like auto rollover based on size limit, ran into this problem when I turned on warm phase, where the shards are shrink to 1, switch alias to new indices (shrink-sam--000110) and delete the old indices (sam--000110).
Hope this stack trace helps, thank you.
{
"indices" : {
"sam--000110" : {
"index" : "sam--000110",
"managed" : true,
"policy" : "sam-lifecycle-policy",
"lifecycle_date_millis" : 1617187196742,
"age" : "12.46d",
"phase" : "warm",
"phase_time_millis" : 1618245496182,
"action" : "shrink",
"action_time_millis" : 1617400022515,
"step" : "ERROR",
"step_time_millis" : 1618245496400,
"failed_step" : "aliases",
"is_auto_retryable_error" : false,
"step_info" : {
"type" : "security_exception",
"reason" : "action [indices:admin/aliases] is unauthorized for API key id [Secret-Key] of user [xpackBasic]",
"stack_trace" : """ElasticsearchSecurityException[action [indices:admin/aliases] is unauthorized for API key id [Secret-Key] of user [xpackBasic]]
at org.elasticsearch.xpack.core.security.support.Exceptions.authorizationError(Exceptions.java:34)
at org.elasticsearch.xpack.security.authz.AuthorizationService.denialException(AuthorizationService.java:609)
at org.elasticsearch.xpack.security.authz.AuthorizationService.access$300(AuthorizationService.java:98)
at org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.handleFailure(AuthorizationService.java:660)
at org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:646)
at org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:616)
at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43)
at org.elasticsearch.xpack.security.authz.RBACEngine.buildIndicesAccessControl(RBACEngine.java:555)
at org.elasticsearch.xpack.security.authz.RBACEngine.lambda$authorizeIndexAction$4(RBACEngine.java:335)
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
at org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.lambda$getAsync$0(AuthorizationService.java:678)
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
at org.elasticsearch.xpack.security.authz.AuthorizationService.resolveIndexNames(AuthorizationService.java:579)
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$6(AuthorizationService.java:273)
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
at org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.lambda$getAsync$0(AuthorizationService.java:678)
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
at org.elasticsearch.xpack.security.authz.RBACEngine.loadAuthorizedIndices(RBACEngine.java:366)
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$5(AuthorizationService.java:269)
at org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.getAsync(AuthorizationService.java:676)
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$8(AuthorizationService.java:272)
at org.elasticsearch.xpack.security.authz.AuthorizationService$CachingAsyncSupplier.getAsync(AuthorizationService.java:676)
at org.elasticsearch.xpack.security.authz.RBACEngine.lambda$authorizeIndexAction$5(RBACEngine.java:327)
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
at org.elasticsearch.xpack.security.authz.RBACEngine.authorizeIndexActionName(RBACEngine.java:351)
at org.elasticsearch.xpack.security.authz.RBACEngine.authorizeIndexAction(RBACEngine.java:324)
at org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:283)
at org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:248)
at org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:212)
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43)
at org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:126)
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
at org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.lambda$getRoles$7(CompositeRolesStore.java:235)
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
at org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.buildAndCacheRoleForApiKey(CompositeRolesStore.java:335)
at org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.lambda$getRoles$8(CompositeRolesStore.java:234)
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
at org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.buildAndCacheRoleForApiKey(CompositeRolesStore.java:335)
at org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:229)
at org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:132)
at org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:120)
at org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:214)
at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.authorizeRequest(SecurityActionFilter.java:173)
at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$3(SecurityActionFilter.java:159)
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:323)
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:384)
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:395)
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:320)
at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:261)
at org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:156)
at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.applyInternal(SecurityActionFilter.java:156)
at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.apply(SecurityActionFilter.java:108)
at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:177)
at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:155)
at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:83)
at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:86)
at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:75)
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:412)
at org.elasticsearch.xpack.core.ClientHelper.executeWithHeadersAsync(ClientHelper.java:177)
at org.elasticsearch.xpack.ilm.LifecyclePolicySecurityClient.doExecute(LifecyclePolicySecurityClient.java:51)
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:412)
at org.elasticsearch.client.support.AbstractClient$IndicesAdmin.execute(AbstractClient.java:1292)
at org.elasticsearch.client.support.AbstractClient$IndicesAdmin.aliases(AbstractClient.java:1338)
at org.elasticsearch.xpack.core.ilm.SwapAliasesAndDeleteSourceIndexStep.deleteSourceIndexAndTransferAliases(SwapAliasesAndDeleteSourceIndexStep.java:90)
at org.elasticsearch.xpack.core.ilm.ShrinkSetAliasStep.performDuringNoSnapshot(ShrinkSetAliasStep.java:39)
at org.elasticsearch.xpack.core.ilm.AsyncRetryDuringSnapshotActionStep.performAction(AsyncRetryDuringSnapshotActionStep.java:40)
at org.elasticsearch.xpack.ilm.IndexLifecycleRunner.maybeRunAsyncAction(IndexLifecycleRunner.java:294)
at org.elasticsearch.xpack.ilm.IndexLifecycleService.maybeRunAsyncAction(IndexLifecycleService.java:90)
at org.elasticsearch.xpack.ilm.action.TransportRetryAction$1.clusterStateProcessed(TransportRetryAction.java:79)
at org.elasticsearch.cluster.service.MasterService$SafeClusterStateTaskListener.clusterStateProcessed(MasterService.java:534)
at org.elasticsearch.cluster.service.MasterService$TaskOutputs.lambda$processedDifferentClusterState$1(MasterService.java:421)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1541)
at org.elasticsearch.cluster.service.MasterService$TaskOutputs.processedDifferentClusterState(MasterService.java:421)
at org.elasticsearch.cluster.service.MasterService.onPublicationSuccess(MasterService.java:281)
at org.elasticsearch.cluster.service.MasterService.publish(MasterService.java:273)
at org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:250)
at org.elasticsearch.cluster.service.MasterService.access$000(MasterService.java:73)
at org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:151)
at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150)
at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188)
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:678)
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252)
at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
"""
},
"phase_execution" : {
"policy" : "sam-lifecycle-policy",
"phase_definition" : {
"min_age" : "2d",
"actions" : {
"forcemerge" : {
"max_num_segments" : 1
},
"set_priority" : {
"priority" : 90
},
"shrink" : {
"number_of_shards" : 1
}
}
},
"version" : 17,
"modified_date_in_millis" : 1617891354954
}
}
}
}
Apache, Apache Lucene, Apache Hadoop, Hadoop, HDFS and the yellow elephant
logo are trademarks of the
Apache Software Foundation
in the United States and/or other countries.