Logs after removing es from blacklist
2017-06-14 08:56:59,431 INFO curator.utils restore_check:1445 _recovery returned an empty response. Trying again.
2017-06-14 08:56:59,431 DEBUG curator.utils wait_for_it:1584 Response: False
2017-06-14 08:56:59,431 DEBUG curator.utils wait_for_it:1604 Action "restore" not yet complete, 9 total seconds elapsed. Waiting 9 seconds before checking again.
2017-06-14 08:57:08,440 DEBUG curator.utils wait_for_it:1581 Elapsed time: 9 seconds
2017-06-14 08:57:08,441 DEBUG urllib3.util.retry from_int:200 Converted retries value: False -> Retry(total=False, connect=None, read=None, redirect=0, status=None)
2017-06-14 08:57:08,447 DEBUG urllib3.connectionpool _make_request:395 http://127.0.0.1:9200 "GET /shopper_actions_2017_06_*/_recovery?human=true HTTP/1.1" 200 4407
2017-06-14 08:57:08,447 INFO elasticsearch log_request_success:83 GET http://127.0.0.1:9200/shopper_actions_2017_06_*/_recovery?human=true [status:200 request:0.007s]
2017-06-14 08:57:08,447 DEBUG elasticsearch log_request_success:85 > None
2017-06-14 08:57:08,448 DEBUG elasticsearch log_request_success:86 < {"shopper_actions_2017_06_13":{"shards":[{"id":1,"type":"SNAPSHOT","stage":"INDEX","primary":true,"start_time":"2017-06-14T08:56:59.443Z","start_time_in_millis":1497430619443,"total_time":"9s","total_time_in_millis":9000,"source":{"repository":"sl_es_s3_repo","snapshot":"curator-20170613215206","version":"5.0.2","index":"shopper_actions_2017_06_13"},"target":{"id":"xzcHEUF7QiaJ1cJgu84Iuw","host":"in-dev01","transport_address":"127.0.0.1:9300","ip":"127.0.0.1","name":"xzcHEUF"},"index":{"size":{"total":"389.8mb","total_in_bytes":408825173,"reused":"0b","reused_in_bytes":0,"recovered":"55.5mb","recovered_in_bytes":58201183,"percent":"14.2%"},"files":{"total":60,"reused":0,"recovered":1,"percent":"1.7%"},"total_time":"8.9s","total_time_in_millis":8991,"source_throttle_time":"-1","source_throttle_time_in_millis":0,"target_throttle_time":"-1","target_throttle_time_in_millis":0},"translog":{"recovered":0,"total":0,"percent":"100.0%","total_on_start":0,"total_time":"0s","total_time_in_millis":0},"verify_index":{"check_index_time":"0s","check_index_time_in_millis":0,"total_time":"0s","total_time_in_millis":0}},{"id":0,"type":"SNAPSHOT","stage":"INDEX","primary":true,"start_time":"2017-06-14T08:56:59.452Z","start_time_in_millis":1497430619452,"total_time":"8.9s","total_time_in_millis":8992,"source":{"repository":"test-repo","snapshot":"curator-20170613215206","version":"5.0.2","index":"shopper_actions_2017_06_13"},"target":{"id":"xzcHEUF7QiaJ1cJgu84Iuw","host":"in-dev01","transport_address":"127.0.0.1:9300","ip":"127.0.0.1","name":"xzcHEUF"},"index":{"size":{"total":"387.8mb","total_in_bytes":406697088,"reused":"0b","reused_in_bytes":0,"recovered":"44.9mb","recovered_in_bytes":47173426,"percent":"11.6%"},"files":{"total":72,"reused":0,"recovered":46,"percent":"63.9%"},"total_time":"8.9s","total_time_in_millis":8987,"source_throttle_time":"-1","source_throttle_time_in_millis":0,"target_throttle_time":"-1","target_throttle_time_in_millis":0},"translog":{"recovered":0,"total":0,"percent":"100.0%","total_on_start":0,"total_time":"0s","total_time_in_millis":0},"verify_index":{"check_index_time":"0s","check_index_time_in_millis":0,"total_time":"0s","total_time_in_millis":0}}]},"shopper_actions_2017_06_12":{"shards":[{"id":1,"type":"SNAPSHOT","stage":"INDEX","primary":true,"start_time":"2017-06-14T08:56:59.457Z","start_time_in_millis":1497430619457,"total_time":"8.9s","total_time_in_millis":8987,"source":{"repository":"test-repo","snapshot":"curator-20170613215206","version":"5.0.2","index":"shopper_actions_2017_06_12"},"target":{"id":"xzcHEUF7QiaJ1cJgu84Iuw","host":"in-dev01","transport_address":"127.0.0.1:9300","ip":"127.0.0.1","name":"xzcHEUF"},"index":{"size":{"total":"645.4mb","total_in_bytes":676801972,"reused":"0b","reused_in_bytes":0,"recovered":"56.9mb","recovered_in_bytes":59734671,"percent":"8.8%"},"files":{"total":94,"reused":0,"recovered":2,"percent":"2.1%"},"total_time":"8.9s","total_time_in_millis":8981,"source_throttle_time":"-1","source_throttle_time_in_millis":0,"target_throttle_time":"-1","target_throttle_time_in_millis":0},"translog":{"recovered":0,"total":0,"percent":"100.0%","total_on_start":0,"total_time":"0s","total_time_in_millis":0},"verify_index":{"check_index_time":"0s","check_index_time_in_millis":0,"total_time":"0s","total_time_in_millis":0}},{"id":0,"type":"SNAPSHOT","stage":"INDEX","primary":true,"start_time":"2017-06-14T08:56:59.464Z","start_time_in_millis":1497430619464,"total_time":"8.9s","total_time_in_millis":8981,"source":{"repository":"sl_es_s3_repo","snapshot":"curator-20170613215206","version":"5.0.2","index":"shopper_actions_2017_06_12"},"target":{"id":"xzcHEUF7QiaJ1cJgu84Iuw","host":"in-dev01","transport_address":"127.0.0.1:9300","ip":"127.0.0.1","name":"xzcHEUF"},"index":{"size":{"total":"646.6mb","total_in_bytes":678057972,"reused":"0b","reused_in_bytes":0,"recovered":"54.6mb","recovered_in_bytes":57299790,"percent":"8.5%"},"files":{"total":90,"reused":0,"recovered":6,"percent":"6.7%"},"total_time":"8.9s","total_time_in_millis":8965,"source_throttle_time":"-1","source_throttle_time_in_millis":0,"target_throttle_time":"-1","target_throttle_time_in_millis":0},"translog":{"recovered":0,"total":0,"percent":"100.0%","total_on_start":0,"total_time":"0s","total_time_in_millis":0},"verify_index":{"check_index_time":"0s","check_index_time_in_millis":0,"total_time":"0s","total_time_in_millis":0}}]}}
2017-06-14 08:57:08,450 ERROR curator.cli cli:193 Failed to complete action: restore. <class 'curator.exceptions.FailedExecution'>: Exception encountered. Rerun with loglevel DEBUG and/or check Elasticsearch logs for more information. Exception: 'shopper_actions_2017_06_*'
