Hi everyone.
I am encountering an error on Elasticsearch 7.17.3 deployed with k8s where the shard assignment fails for Kibana_task_manager.
GET _cat/shards?v&h=index,shard,prirep,state,unassigned.reason&s=state:asc
index shard prirep state unassigned.reason
.kibana_task_manager_7.17.3_001 0 p UNASSIGNED ALLOCATION_FAILED
GET _cluster/allocation/explain?pretty
{
"note" : "No shard was specified in the explain API request, so this response explains a randomly chosen unassigned shard. There may be other unassigned shards in this cluster which cannot be assigned for different reasons. It may not be possible to assign this shard until one of the other shards is assigned correctly. To explain the allocation of other shards (whether assigned or unassigned) you must specify the target shard in the request to this API.",
"index" : ".kibana_task_manager_7.17.3_001",
"shard" : 0,
"primary" : true,
"current_state" : "unassigned",
"unassigned_info" : {
"reason" : "ALLOCATION_FAILED",
"at" : "2023-01-06T17:39:14.402Z",
"failed_allocation_attempts" : 1,
"details" : """failed shard on node [B3Ilx0zjR7OS60Bd3-QxyA]: shard failure, reason [refresh failed source[api]], failure CorruptIndexException[compound sub-files must have a valid codec header and footer: file is too small (0 bytes) (resource=BufferedChecksumIndexInput(MMapIndexInput(path="/usr/share/elasticsearch/data/nodes/0/indices/fzylNJyUTD2fNgJu0Nnicg/0/index/_f96_Lucene84_0.tip")))]""",
"last_allocation_status" : "no_valid_shard_copy"
},
"can_allocate" : "no_valid_shard_copy",
"allocate_explanation" : "cannot allocate because all found copies of the shard are either stale or corrupt",
"node_allocation_decisions" : [
{
"node_id" : "B3Ilx0zjR7OS60Bd3-QxyA",
"node_name" : "elasticsearch-master-0",
"transport_address" : "10.192.1.126:9300",
"node_attributes" : {
"ml.machine_memory" : "2147483648",
"xpack.installed" : "true",
"transform.node" : "true",
"ml.max_open_jobs" : "512",
"ml.max_jvm_size" : "1073741824"
},
"node_decision" : "no",
"store" : {
"in_sync" : true,
"allocation_id" : "0l3AwBWNRLWuyQwDgLqK0A",
"store_exception" : {
"type" : "corrupt_index_exception",
"reason" : "failed engine (reason: [refresh failed source[api]]) (resource=preexisting_corruption)",
"caused_by" : {
"type" : "i_o_exception",
"reason" : "failed engine (reason: [refresh failed source[api]])",
"caused_by" : {
"type" : "corrupt_index_exception",
"reason" : """compound sub-files must have a valid codec header and footer: file is too small (0 bytes) (resource=BufferedChecksumIndexInput(MMapIndexInput(path="/usr/share/elasticsearch/data/nodes/0/indices/fzylNJyUTD2fNgJu0Nnicg/0/index/_f96_Lucene84_0.tip")))"""
}
}
}
}
}
]
}
I have performed manual shard reassignment, but it has not resolved the issue.
Is there any solution?
Appreciate any help.
POST _cluster/reroute?retry_failed
{
"note" : "No shard was specified in the explain API request, so this response explains a randomly chosen unassigned shard. There may be other unassigned shards in this cluster which cannot be assigned for different reasons. It may not be possible to assign this shard until one of the other shards is assigned correctly. To explain the allocation of other shards (whether assigned or unassigned) you must specify the target shard in the request to this API.",
"index" : ".kibana_task_manager_7.17.3_001",
"shard" : 0,
"primary" : true,
"current_state" : "unassigned",
"unassigned_info" : {
"reason" : "MANUAL_ALLOCATION",
"at" : "2023-01-11T13:43:18.753Z",
"details" : """failed shard on node [yQQPYsL3QhaMeHl21FvvWQ]: shard failure, reason [refresh failed source[api]], failure CorruptIndexException[compound sub-files must have a valid codec header and footer: file is too small (0 bytes) (resource=BufferedChecksumIndexInput(NIOFSIndexInput(path="/usr/share/elasticsearch/data/nodes/0/indices/hX2STRqsRdGnvN5gzciUyQ/0/index/_8f9_Lucene84_0.tmd")))]""",
"last_allocation_status" : "no_valid_shard_copy"
},
"can_allocate" : "no_valid_shard_copy",
"allocate_explanation" : "cannot allocate because all found copies of the shard are either stale or corrupt",
"node_allocation_decisions" : [
{
"node_id" : "yQQPYsL3QhaMeHl21FvvWQ",
"node_name" : "elasticsearch-master-0",
"transport_address" : "10.192.4.149:9300",
"node_attributes" : {
"ml.machine_memory" : "2147483648",
"xpack.installed" : "true",
"transform.node" : "true",
"ml.max_open_jobs" : "512",
"ml.max_jvm_size" : "1073741824"
},
"node_decision" : "no",
"store" : {
"in_sync" : true,
"allocation_id" : "0zZRq-lJTpaFMgsPmlUelQ",
"store_exception" : {
"type" : "corrupt_index_exception",
"reason" : "failed engine (reason: [refresh failed source[api]]) (resource=preexisting_corruption)",
"caused_by" : {
"type" : "i_o_exception",
"reason" : "failed engine (reason: [refresh failed source[api]])",
"caused_by" : {
"type" : "corrupt_index_exception",
"reason" : """compound sub-files must have a valid codec header and footer: file is too small (0 bytes) (resource=BufferedChecksumIndexInput(NIOFSIndexInput(path="/usr/share/elasticsearch/data/nodes/0/indices/hX2STRqsRdGnvN5gzciUyQ/0/index/_8f9_Lucene84_0.tmd")))"""
}
}
}
}
}
]
}