Hello all ,
im using elasticsearch 1.3.4 with 5 data nodes + 3 masters
trying to do snapshot to one of my indices
snapshot partially succeeded.
i get this output:
"failures": [
{
"node_id": "XYVA5mIUSPKovhchtps8",
"index": "XXX",
"reason": "IndexShardSnapshotFailedException[[XXX][59] Failed to perform snapshot (index files)]; nested: FileNotFoundException[/var/BackupEL/my_backup/indices/XXX/59/__2 (No such file or directory)]; ",
"shard_id": 59,
"status": "INTERNAL_SERVER_ERROR"
},
{
"node_id": "XYVA5mIUSPKovhchtps8",
"index": "XXX",
"reason": "IndexShardSnapshotFailedException[[XXX][56] Failed to perform snapshot (index files)]; nested: FileNotFoundException[/var/BackupEL/my_backup/indices/XXX/56/__0 (No such file or directory)]; ",
"shard_id": 56,
"status": "INTERNAL_SERVER_ERROR"
},
{
"node_id": "XYVA5mIUSPKovhchtps8",
"index": "XXX",
"reason": "IndexShardSnapshotFailedException[[XXX][55] Failed to perform snapshot (index files)]; nested: FileNotFoundException[/var/BackupEL/my_backup/indices/XXX/55/__0 (No such file or directory)]; ",
"shard_id": 55,
"status": "INTERNAL_SERVER_ERROR"
},
{
"node_id": "XYVA5mIUSPKovhchtps8",
"index": "XXX",
"reason": "IndexShardSnapshotFailedException[[XXX][52] Failed to perform snapshot (index files)]; nested: FileNotFoundException[/var/BackupEL/my_backup/indices/XXX/52/__0 (No such file or directory)]; ",
"shard_id": 52,
"status": "INTERNAL_SERVER_ERROR"
},
{
"node_id": "v9NvVnkESgCG3NfhIJi",
"index": "XXX",
"reason": "IndexShardSnapshotFailedException[[XXX][13] Failed to perform snapshot (index files)]; nested: FileNotFoundException[/var/BackupEL/my_backup/indices/XXX/13/__1 (No such file or directory)]; ",
"shard_id": 13,
"status": "INTERNAL_SERVER_ERROR"
},
{
"node_id": "v9NvVnkESgCG3NfhIJi",
"index": "XXX",
"reason": "IndexShardSnapshotFailedException[[XXX][16] Failed to perform snapshot (index files)]; nested: FileNotFoundException[/var/BackupEL/my_backup/indices/XXX/16/__1 (No such file or directory)]; ",
"shard_id": 16,
"status": "INTERNAL_SERVER_ERROR"
},
{
"node_id": "XYVA5mIUSPKovhchtps8",
"index": "XXX",
"reason": "IndexShardSnapshotFailedException[[XXX][7] Failed to perform snapshot (index files)]; nested: FileNotFoundException[/var/BackupEL/my_backup/indices/XXX/7/__0 (No such file or directory)]; ",
"shard_id": 7,
"status": "INTERNAL_SERVER_ERROR"
},
{
"node_id": "XYVA5mIUSPKovhchtps8",
"index": "XXX",
"reason": "IndexShardSnapshotFailedException[[XXX][30] Failed to perform snapshot (index files)]; nested: FileNotFoundException[/var/BackupEL/my_backup/indices/XXX/30/__0 (No such file or directory)]; ",
"shard_id": 30,
"status": "INTERNAL_SERVER_ERROR"
},
{
"node_id": "v9NvVnkESgCG3NfhIJi",
"index": "XXX",
"reason": "IndexShardSnapshotFailedException[[XXX][29] Failed to perform snapshot (index files)]; nested: FileNotFoundException[/var/BackupEL/my_backup/indices/XXX/29/__0 (No such file or directory)]; ",
"shard_id": 29,
"status": "INTERNAL_SERVER_ERROR"
},
{
"node_id": "v9NvVnkESgCG3NfhIJi",
"index": "XXX",
"reason": "IndexShardSnapshotFailedException[[XXX][32] Failed to perform snapshot (index files)]; nested: FileNotFoundException[/var/BackupEL/my_backup/indices/XXX/32/__0 (No such file or directory)]; ",
"shard_id": 32,
"status": "INTERNAL_SERVER_ERROR"
},
{
"node_id": "XYVA5mIUSPKovhchtps8",
"index": "XXX",
"reason": "IndexShardSnapshotFailedException[[XXX][24] Failed to perform snapshot (index files)]; nested: FileNotFoundException[/var/BackupEL/my_backup/indices/XXX/24/__0 (No such file or directory)]; ",
"shard_id": 24,
"status": "INTERNAL_SERVER_ERROR"
},
{
"node_id": "XYVA5mIUSPKovhchtps8",
"index": "XXX",
"reason": "IndexShardSnapshotFailedException[[XXX][17] Failed to perform snapshot (index files)]; nested: FileNotFoundException[/var/BackupEL/my_backup/indices/XXX/17/__0 (No such file or directory)]; ",
"shard_id": 17,
"status": "INTERNAL_SERVER_ERROR"
},
{
"node_id": "XYVA5mIUSPKovhchtps8",
"index": "XXX",
"reason": "IndexShardSnapshotFailedException[[XXX][36] Failed to perform snapshot (index files)]; nested: FileNotFoundException[/var/BackupEL/my_backup/indices/XXX/36/__0 (No such file or directory)]; ",
"shard_id": 36,
"status": "INTERNAL_SERVER_ERROR"
}
],
"shards": {
"total": 60,
"failed": 13,
"successful": 47
}
}
]
i can see that the failures occur on 2 of the data nodes
any idea why ?
btw , its not permission issue
thanks a lot