Hello,
I have 2 ES9.0.4 with exactily the same configuration and setup.
In ES1: I created one index and put some dummy data as:
get test/_search
{ "took": 0, "timed_out": false, "_shards": { "total": 1, "successful": 1, "skipped": 0, "failed": 0 }, "hits": { "total": { "value": 1, "relation": "eq" }, "max_score": 1, "hits": [ { "_index": "test", "_id": "2R9hqJgB8CDT6OO9Ckeq", "_score": 1, "_source": { "name": "foo", "title": "bar" } } ] } }
So far so good, I made a folder and defined as repository. Created the snapshot that can be recovered back correctly. When I delete and recover the index in the same ES, I got yellow status which is fine because no replicate is there.
ES2: Now I add the same repository in ES2 and enable it with:
PUT _snapshot/from_cold_backup { "type": "fs", "settings": { "location": "/mnt/extvol/es_repo", "readonly": false }}
To verify that I used:
POST /_snapshot/from_cold_backup/_verify
{ "nodes": { "aZQG-MvtRc-C4OeZmXWM9g": { "name": "es2" } }}
According to How to export/import an ES snapshot to/from an external harddrive and Snapshot and Restore in readable format I can copy the entire repository (including the index files) to a new ES as a repository.
Therefore I copied the entire folder of the repository which includes these files:
index-2 meta-vm3eo9OmSlGfFBdGnSeV9Q.dat
index.latest snap-vm3eo9OmSlGfFBdGnSeV9Q.dat
indices
from ES1 to ES2 /mnt/extvol/es_repofolder. After one restart ES2 service, the snapshot appears in the Kibana page. GET /_snapshot/from_cold_backup/_allalso shows the snapshot name.
When I restore the snapshot either using kibana > Stack Management>Snapshot and Restore>Snapshots or using command POST /_snapshot/from_cold_backup/metadata_0aa_list_cold-kving1xwrfcu3dfwsvtscw/_restorealthogh there is no error, the response is { "accepted": true } but the status remains red.
Any request returns
{ "error": { "root_cause": [ { "type": "no_shard_available_action_exception", "reason": null } ], "type": "search_phase_execution_exception", "reason": "all shards failed", "phase": "query", "grouped": true, "failed_shards": [ { "shard": 0, "index": "test", "node": null, "reason": { "type": "no_shard_available_action_exception", "reason": null } } ] }, "status": 503 }
according to ERROR: no_shard_available_action_exception after restoring the snapshot
the watermark is not the issue /dev/vdd 984G 612G 322G 66% /mnt/extvol
Here is a full log of
GET _snapshot/from_cold_backup/metadata_0aa_list_cold-kving1xwrfcu3dfwsvtscw/_status
{
"snapshots": [
{
"snapshot": "metadata_0aa_list_cold-kving1xwrfcu3dfwsvtscw",
"repository": "from_cold_backup",
"uuid": "vm3eo9OmSlGfFBdGnSeV9Q",
"state": "SUCCESS",
"include_global_state": true,
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 17,
"failed": 0,
"total": 17
},
"stats": {
"incremental": {
"file_count": 212,
"size_in_bytes": 3879192
},
"total": {
"file_count": 212,
"size_in_bytes": 3879192
},
"start_time_in_millis": 1755171701737,
"time_in_millis": 2401
},
"indices": {
".kibana_usage_counters_8.16.0_001": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 25,
"size_in_bytes": 107691
},
"total": {
"file_count": 25,
"size_in_bytes": 107691
},
"start_time_in_millis": 1755171702738,
"time_in_millis": 200
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 25,
"size_in_bytes": 107691
},
"total": {
"file_count": 25,
"size_in_bytes": 107691
},
"start_time_in_millis": 1755171702738,
"time_in_millis": 200
}
}
}
},
".secrets-inference": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 10,
"size_in_bytes": 9654
},
"total": {
"file_count": 10,
"size_in_bytes": 9654
},
"start_time_in_millis": 1755171703138,
"time_in_millis": 0
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 10,
"size_in_bytes": 9654
},
"total": {
"file_count": 10,
"size_in_bytes": 9654
},
"start_time_in_millis": 1755171703138,
"time_in_millis": 0
}
}
}
},
"test": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 4,
"size_in_bytes": 5528
},
"total": {
"file_count": 4,
"size_in_bytes": 5528
},
"start_time_in_millis": 1755171703738,
"time_in_millis": 0
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 4,
"size_in_bytes": 5528
},
"total": {
"file_count": 4,
"size_in_bytes": 5528
},
"start_time_in_millis": 1755171703738,
"time_in_millis": 0
}
}
}
},
".apm-agent-configuration": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 1,
"size_in_bytes": 250
},
"total": {
"file_count": 1,
"size_in_bytes": 250
},
"start_time_in_millis": 1755171703538,
"time_in_millis": 0
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 1,
"size_in_bytes": 250
},
"total": {
"file_count": 1,
"size_in_bytes": 250
},
"start_time_in_millis": 1755171703538,
"time_in_millis": 0
}
}
}
},
".kibana_security_session_1": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 7,
"size_in_bytes": 14022
},
"total": {
"file_count": 7,
"size_in_bytes": 14022
},
"start_time_in_millis": 1755171703338,
"time_in_millis": 0
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 7,
"size_in_bytes": 14022
},
"total": {
"file_count": 7,
"size_in_bytes": 14022
},
"start_time_in_millis": 1755171703338,
"time_in_millis": 0
}
}
}
},
".kibana_analytics_8.16.0_001": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 13,
"size_in_bytes": 2467923
},
"total": {
"file_count": 13,
"size_in_bytes": 2467923
},
"start_time_in_millis": 1755171702938,
"time_in_millis": 200
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 13,
"size_in_bytes": 2467923
},
"total": {
"file_count": 13,
"size_in_bytes": 2467923
},
"start_time_in_millis": 1755171702938,
"time_in_millis": 200
}
}
}
},
".kibana_entities-definitions-1": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 16,
"size_in_bytes": 31266
},
"total": {
"file_count": 16,
"size_in_bytes": 31266
},
"start_time_in_millis": 1755171702738,
"time_in_millis": 0
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 16,
"size_in_bytes": 31266
},
"total": {
"file_count": 16,
"size_in_bytes": 31266
},
"start_time_in_millis": 1755171702738,
"time_in_millis": 0
}
}
}
},
".tasks": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 28,
"size_in_bytes": 93951
},
"total": {
"file_count": 28,
"size_in_bytes": 93951
},
"start_time_in_millis": 1755171703538,
"time_in_millis": 200
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 28,
"size_in_bytes": 93951
},
"total": {
"file_count": 28,
"size_in_bytes": 93951
},
"start_time_in_millis": 1755171703538,
"time_in_millis": 200
}
}
}
},
".kibana_alerting_cases_8.16.0_001": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 4,
"size_in_bytes": 7139
},
"total": {
"file_count": 4,
"size_in_bytes": 7139
},
"start_time_in_millis": 1755171703138,
"time_in_millis": 0
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 4,
"size_in_bytes": 7139
},
"total": {
"file_count": 4,
"size_in_bytes": 7139
},
"start_time_in_millis": 1755171703138,
"time_in_millis": 0
}
}
}
},
".security-7": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 13,
"size_in_bytes": 656344
},
"total": {
"file_count": 13,
"size_in_bytes": 656344
},
"start_time_in_millis": 1755171702137,
"time_in_millis": 0
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 13,
"size_in_bytes": 656344
},
"total": {
"file_count": 13,
"size_in_bytes": 656344
},
"start_time_in_millis": 1755171702137,
"time_in_millis": 0
}
}
}
},
".kibana_8.16.0_001": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 22,
"size_in_bytes": 96954
},
"total": {
"file_count": 22,
"size_in_bytes": 96954
},
"start_time_in_millis": 1755171703938,
"time_in_millis": 0
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 22,
"size_in_bytes": 96954
},
"total": {
"file_count": 22,
"size_in_bytes": 96954
},
"start_time_in_millis": 1755171703938,
"time_in_millis": 0
}
}
}
},
".apm-custom-link": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 1,
"size_in_bytes": 250
},
"total": {
"file_count": 1,
"size_in_bytes": 250
},
"start_time_in_millis": 1755171702938,
"time_in_millis": 0
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 1,
"size_in_bytes": 250
},
"total": {
"file_count": 1,
"size_in_bytes": 250
},
"start_time_in_millis": 1755171702938,
"time_in_millis": 0
}
}
}
},
".kibana_ingest_8.16.0_001": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 7,
"size_in_bytes": 239381
},
"total": {
"file_count": 7,
"size_in_bytes": 239381
},
"start_time_in_millis": 1755171702938,
"time_in_millis": 0
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 7,
"size_in_bytes": 239381
},
"total": {
"file_count": 7,
"size_in_bytes": 239381
},
"start_time_in_millis": 1755171702938,
"time_in_millis": 0
}
}
}
},
".security-profile-8": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 4,
"size_in_bytes": 8966
},
"total": {
"file_count": 4,
"size_in_bytes": 8966
},
"start_time_in_millis": 1755171704138,
"time_in_millis": 0
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 4,
"size_in_bytes": 8966
},
"total": {
"file_count": 4,
"size_in_bytes": 8966
},
"start_time_in_millis": 1755171704138,
"time_in_millis": 0
}
}
}
},
".inference": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 10,
"size_in_bytes": 17654
},
"total": {
"file_count": 10,
"size_in_bytes": 17654
},
"start_time_in_millis": 1755171703338,
"time_in_millis": 0
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 10,
"size_in_bytes": 17654
},
"total": {
"file_count": 10,
"size_in_bytes": 17654
},
"start_time_in_millis": 1755171703338,
"time_in_millis": 0
}
}
}
},
".kibana_security_solution_8.16.0_001": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 13,
"size_in_bytes": 46100
},
"total": {
"file_count": 13,
"size_in_bytes": 46100
},
"start_time_in_millis": 1755171702738,
"time_in_millis": 200
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 13,
"size_in_bytes": 46100
},
"total": {
"file_count": 13,
"size_in_bytes": 46100
},
"start_time_in_millis": 1755171702738,
"time_in_millis": 200
}
}
}
},
".kibana_task_manager_8.16.0_001": {
"shards_stats": {
"initializing": 0,
"started": 0,
"finalizing": 0,
"done": 1,
"failed": 0,
"total": 1
},
"stats": {
"incremental": {
"file_count": 34,
"size_in_bytes": 76119
},
"total": {
"file_count": 34,
"size_in_bytes": 76119
},
"start_time_in_millis": 1755171702538,
"time_in_millis": 0
},
"shards": {
"0": {
"stage": "DONE",
"stats": {
"incremental": {
"file_count": 34,
"size_in_bytes": 76119
},
"total": {
"file_count": 34,
"size_in_bytes": 76119
},
"start_time_in_millis": 1755171702538,
"time_in_millis": 0
}
}
}
}
}
}
]
}