es version: 6.2.4
hadoop version: 2.6.0-cdh5.11.1
I want make a mapreduce with es use Es-Hadoop, however can not connect es cluster when I install X-pack for es.
// mapreduce conf
conf.setBoolean("mapred.map.tasks.speculative.execution", false);
conf.setBoolean("mapred.reduce.tasks.speculative.execution", false);
conf.set("es.net.http.auth.user", "xxxx");
conf.set("es.net.http.auth.pass", "xxxx");
conf.set("es.nodes", "xxxx:9200");
// error logs
2018-12-13 10:55:17,834 INFO [main] org.elasticsearch.hadoop.util.Version: Elasticsearch Hadoop v6.2.4 [0dadc1ea14]
2018-12-13 10:55:18,095 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: I/O exception (java.net.ConnectException) caught when processing request: Connection refused (Connection refused)
2018-12-13 10:55:18,095 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: Retrying request
2018-12-13 10:55:18,096 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: I/O exception (java.net.ConnectException) caught when processing request: Connection refused (Connection refused)
2018-12-13 10:55:18,096 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: Retrying request
2018-12-13 10:55:18,097 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: I/O exception (java.net.ConnectException) caught when processing request: Connection refused (Connection refused)
2018-12-13 10:55:18,097 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: Retrying request
2018-12-13 10:55:18,099 ERROR [main] org.elasticsearch.hadoop.rest.NetworkClient: Node [10.174.20.34:9200] failed (Connection refused (Connection refused)); selected next node [xxxx:9200]
2018-12-13 10:55:18,100 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: I/O exception (java.net.ConnectException) caught when processing request: Connection refused (Connection refused)
2018-12-13 10:55:18,100 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: Retrying request
2018-12-13 10:55:18,101 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: I/O exception (java.net.ConnectException) caught when processing request: Connection refused (Connection refused)
2018-12-13 10:55:18,101 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: Retrying request
2018-12-13 10:55:18,101 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: I/O exception (java.net.ConnectException) caught when processing request: Connection refused (Connection refused)
2018-12-13 10:55:18,101 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: Retrying request
2018-12-13 10:55:18,103 ERROR [main] org.elasticsearch.hadoop.rest.NetworkClient: Node [xxxx:9200] failed (Connection refused (Connection refused)); selected next node [xxxx:9200]
2018-12-13 10:55:18,105 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: I/O exception (java.net.ConnectException) caught when processing request: Connection refused (Connection refused)
2018-12-13 10:55:18,105 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: Retrying request
2018-12-13 10:55:18,106 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: I/O exception (java.net.ConnectException) caught when processing request: Connection refused (Connection refused)
2018-12-13 10:55:18,106 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: Retrying request
2018-12-13 10:55:18,106 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: I/O exception (java.net.ConnectException) caught when processing request: Connection refused (Connection refused)
2018-12-13 10:55:18,106 INFO [main] org.apache.commons.httpclient.HttpMethodDirector: Retrying request
2018-12-13 10:55:18,107 ERROR [main] org.elasticsearch.hadoop.rest.NetworkClient: Node [xxxx:9200] failed (Connection refused (Connection refused)); no other nodes left - aborting...
2018-12-13 10:55:18,109 WARN [main] org.apache.hadoop.mapred.YarnChild: Exception running child : org.elasticsearch.hadoop.rest.EsHadoopNoNodesLeftException: Connection error (check network and/or proxy settings)- all nodes failed; tried [[xxxx:9200, xxxx:9200, xxxx:9200]]
at org.elasticsearch.hadoop.rest.NetworkClient.execute(NetworkClient.java:149)
at org.elasticsearch.hadoop.rest.RestClient.execute(RestClient.java:380)
at org.elasticsearch.hadoop.rest.RestClient.execute(RestClient.java:344)
at org.elasticsearch.hadoop.rest.RestClient.execute(RestClient.java:348)
at org.elasticsearch.hadoop.rest.RestClient.get(RestClient.java:158)
at org.elasticsearch.hadoop.rest.RestClient.getHttpNodes(RestClient.java:115)
at org.elasticsearch.hadoop.rest.InitializationUtils.discoverNodesIfNeeded(InitializationUtils.java:92)
at org.elasticsearch.hadoop.rest.RestService.createWriter(RestService.java:579)
at org.elasticsearch.hadoop.mr.EsOutputFormat$EsRecordWriter.init(EsOutputFormat.java:173)
at org.elasticsearch.hadoop.mr.EsOutputFormat$EsRecordWriter.write(EsOutputFormat.java:149)
at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.write(ReduceTask.java:558)
at org.apache.hadoop.mapreduce.task.TaskInputOutputContextImpl.write(TaskInputOutputContextImpl.java:89)
at org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer$Context.write(WrappedReducer.java:105)
at org.apache.hadoop.mapreduce.Reducer.reduce(Reducer.java:150)
at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:171)
at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:627)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389)
at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:164)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1920)
at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)