Es docker 10.2 version is upgraded to 12.0, log "SearchPhaseExecutionException: all shards failed"

Elastic Stack Elasticsearch
1 
{"type": "server", "timestamp": "2021-03-27T00:33:09,519Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "es-single-node", "node.name": "es-docker", "message": "path: /elastalert_status_test/_search, params: {size=1000, index=elastalert_status_test}", "cluster.uuid": "V-K8-DD7RValiCtkesXqMg", "node.id": "9_5rNmciQ1CQniRQ1rvFRw" , 

"stacktrace": ["org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:636) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:357) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:669) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:440) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$1(AbstractSearchAsyncAction.java:265) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction$2.doRun(AbstractSearchAsyncAction.java:333) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.12.0.jar:7.12.0]",

"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",

"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",

"at java.lang.Thread.run(Thread.java:832) [?:?]",

"Caused by: org.elasticsearch.action.NoShardAvailableActionException",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:473) ~[elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:422) [elasticsearch-7.12.0.jar:7.12.0]",

"... 9 more"] }

{"type": "server", "timestamp": "2021-03-27T00:33:10,534Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "es-single-node", "node.name": "es-docker", "message": "path: /elastalert_status_test/_search, params: {size=1000, index=elastalert_status_test}", "cluster.uuid": "V-K8-DD7RValiCtkesXqMg", "node.id": "9_5rNmciQ1CQniRQ1rvFRw" , 

"stacktrace": ["org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:636) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:357) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:669) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:440) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$1(AbstractSearchAsyncAction.java:265) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction$2.doRun(AbstractSearchAsyncAction.java:333) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:33) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:732) [elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) [elasticsearch-7.12.0.jar:7.12.0]",

"at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) [?:?]",

"at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) [?:?]",

"at java.lang.Thread.run(Thread.java:832) [?:?]",

"Caused by: org.elasticsearch.action.NoShardAvailableActionException",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:473) ~[elasticsearch-7.12.0.jar:7.12.0]",

"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:422) [elasticsearch-7.12.0.jar:7.12.0]",

"... 9 more"] }

{"type": "deprecation", "timestamp": "2021-03-27T00:33:12,150Z", "level": "DEPRECATION", "component": "o.e.d.a.b.BulkRequestParser", "cluster.name": "es-single-node", "node.name": "es-docker", "message": "[types removal] Specifying types in bulk requests is deprecated.", "cluster.uuid": "V-K8-DD7RValiCtkesXqMg", "node.id": "9_5rNmciQ1CQniRQ1rvFRw"  }

{"type": "server", "timestamp": "2021-03-27T00:33:13,545Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "es-single-node", "node.name": "es-docker", "message": "path: /elastalert_status_test/_search, params: {size=1000, index=elastalert_status_test}", "cluster.uuid": "V-K8-DD7RValiCtkesXqMg", "node.id": "9_5rNmciQ1CQniRQ1rvFRw" , 

"stacktrace": ["org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed",

Complete log content:
https://paste.ubuntu.com/p/M88XBCYTg6/

I just changed the image address of es and kibana.

2

All local indexes are invalid

# curl -XGET 'http://127.0.0.1:9200/_cat/shards?h=index,shard,prirep,state,unassigned.reason'
ack_k8s_dev-2020.12.27                   0 p STARTED    
ack_k8s_dev-2020.12.27                   0 r UNASSIGNED CLUSTER_RECOVERED
ilm-history-2-000007                     0 p STARTED    
ack_k8s_dev-2021.02.13                   0 p STARTED    
ack_k8s_dev-2021.02.13                   0 r UNASSIGNED CLUSTER_RECOVERED
ack_k8s_dev-2021.02.27                   0 p STARTED    
ack_k8s_dev-2021.02.27                   0 r UNASSIGNED CLUSTER_RECOVERED
lonsid_k8s_dev-2021.03.15                0 p STARTED    
lonsid_k8s_dev-2021.03.15                0 r UNASSIGNED CLUSTER_RECOVERED
mysql_2020.09                            0 p STARTED    
mysql_2020.09                            0 r UNASSIGNED CLUSTER_RECOVERED
ack_k8s_dev-2021.03.15                   0 p STARTED    
ack_k8s_dev-2021.03.15                   0 r UNASSIGNED CLUSTER_RECOVERED
ack_k8s_dev-2021.03.13                   0 p STARTED    
ack_k8s_dev-2021.03.13                   0 r UNASSIGNED CLUSTER_RECOVERED
......

curl -sXGET 'http://localhost:9200/_cluster/health?pretty'
{
  "cluster_name" : "es-single-node",
  "status" : "yellow",
  "timed_out" : false,
  "number_of_nodes" : 1,
  "number_of_data_nodes" : 1,
  "active_primary_shards" : 316,
  "active_shards" : 316,
  "relocating_shards" : 0,
  "initializing_shards" : 0,
  "unassigned_shards" : 266,
  "delayed_unassigned_shards" : 0,
  "number_of_pending_tasks" : 0,
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 0,
  "active_shards_percent_as_number" : 54.29553264604811
}
3

kibana logs
https://paste.ubuntu.com/p/hFfmtVDmfh/

4

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.