Hello,
Our developers are using elasticsearch Java bindings to query a Bulk elasticsearch API (org.elasticsearch.action.bulk .BulkRequestBuilder).
The 5-15% of the requests into a bulk request aren't processing on the production (stagging is ok).
How I can found the source of the problem? I tried to put the es.logger.level to INFO but it doesn't helps.
The code is similar to the following:
void executeBuilder(BulkRequestBuilder bulkRequestBuilder)
{
if(bulkRequestBuilder.numberOfActions() == 0)
return;
++stats.flushes;
BulkResponse response = bulkRequestBuilder.execute().actionGet();
if (response.hasFailures()) {
logger.warn("failed to execute" + response.buildFailureMessage());
}
for(BulkItemResponse resp : response){
if(resp.isFailed()){
stats.failed++;
}else{
stats.succeeded++;
}
}
}
the counter of stats.failed ~= 5-15% per bulk request.
PS:
curl http://localhost:9200/_nodes/_local?pretty=true
{
"cluster_name" : "cluster",
"nodes" : {
"xxx" : {
"version" : "2.3.3",
"build" : "218bdf1",
"settings" : {
"node" : {
"data" : "true",
"master" : "true"
},
"indices" : {"cache" : {"query" : {"size" : "5%"}}},
"bootstrap" : {
"mlockall" : "true"
},
"foreground" : "false",
"config" : {"ignore_system_properties" : "true"},
},
"os" : { "refresh_interval_in_millis" : 1000, "allocated_processors" : 16 },
"process" : {
"refresh_interval_in_millis" : 1000,"id" : 45184,"mlockall" : true },
"jvm" : {
"pid" : 45184,
"version" : "1.8.0_72-internal",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "25.72-b05",
"vm_vendor" : "Oracle Corporation",
"start_time_in_millis" : 1470557125542,
"mem" : {
"heap_init_in_bytes" : 34359738368,
"heap_max_in_bytes" : 34246361088,
"non_heap_init_in_bytes" : 2555904,
"non_heap_max_in_bytes" : 0,
"direct_max_in_bytes" : 34246361088
},
"gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
"memory_pools" : [ "Code Cache", "Metaspace", "Par Eden Space", "Par Survivor Space", "CMS Old Gen" ],
"using_compressed_ordinary_object_pointers" : "false"
},
"thread_pool" : {
"bulk" : {
"type" : "fixed",
"min" : 16,
"max" : 16,
"queue_size" : 50
}
},
"plugins" : [ {
"name" : "cloud-aws",
"version" : "2.3.3",
"description" : "The Amazon Web Service (AWS) Cloud plugin allows to use AWS API for the unicast discovery mechanism and add S3 repositories.",
"jvm" : true,
"classname" : "org.elasticsearch.plugin.cloud.aws.CloudAwsPlugin",
"isolated" : true,
"site" : false
} ],
"modules" : [ {
"name" : "lang-expression",
"version" : "2.3.3",
"description" : "Lucene expressions integration for Elasticsearch",
"jvm" : true,
"classname" : "org.elasticsearch.script.expression.ExpressionPlugin",
"isolated" : true,
"site" : false
}, {
"name" : "lang-groovy",
"version" : "2.3.3",
"description" : "Groovy scripting integration for Elasticsearch",
"jvm" : true,
"classname" : "org.elasticsearch.script.groovy.GroovyPlugin",
"isolated" : true,
"site" : false
}, {
"name" : "reindex",
"version" : "2.3.3",
"description" : "_reindex and _update_by_query APIs",
"jvm" : true,
"classname" : "org.elasticsearch.index.reindex.ReindexPlugin",
"isolated" : true,
"site" : false
} ]
}
}
}