When I do search All my shards are failing and I am receiving the below exception. It was working fine till yesterday and all my query's and shard's are continuously failing now
//[2020-03-12T16:30:30,410][DEBUG][o.e.a.s.TransportSearchAction] [scrpelasticsearchproducts] [supplierproductscloudindex1][1], node[2gRMdwnqSV6tjD52FO0wAw], [P], s[STARTED], a[id=Q9_sDxWMQ0uUJRyg40V3Ew]: Failed to execute [SearchRequest{searchType=QUERY_THEN_FETCH, indices=[retailerproductsindex1, supplierproductscloudindex1], indicesOptions=IndicesOptions[id=38, ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, allow_alisases_to_multiple_indices=true, forbid_closed_indices=true], types=[product], routing='null', preference='null', requestCache=null, scroll=null, source={
"query" : {
"bool" : {
"must" : [
{
"bool" : {
"should" : [
{
"match" : {
"nameLower" : {
"query" : "weighted",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 50.0
}
}
},
{
"match" : {
"nameLowerSyn" : {
"query" : "weighted",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 50.0
}
}
},
{
"match" : {
"nameLower" : {
"query" : "glov",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 50.0
}
}
},
{
"match" : {
"nameLowerSyn" : {
"query" : "glov",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 50.0
}
}
},
{
"match" : {
"nameLower" : {
"query" : "weighted glov",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 50.0
}
}
},
{
"match" : {
"nameLowerSyn" : {
"query" : "weighted glov",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 50.0
}
}
},
{
"match" : {
"codeLowerSyn" : {
"query" : "weighted",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 500.0
}
}
},
{
"match" : {
"codeLower" : {
"query" : "weighted",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 500.0
}
}
},
{
"match" : {
"codeLowerSyn" : {
"query" : "glov",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 500.0
}
}
},
{
"match" : {
"codeLower" : {
"query" : "glov",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 500.0
}
}
},
{
"match" : {
"codeLowerSyn" : {
"query" : "weighted glov",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 500.0
}
}
},
{
"match" : {
"codeLower" : {
"query" : "weighted glov",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 500.0
}
}
},
{
"match" : {
"brandNameLower" : {
"query" : "weighted",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 3.0
}
}
},
{
"match" : {
"brandNameLowerSyn" : {
"query" : "weighted",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 3.0
}
}
},
{
"match" : {
"brandNameLower" : {
"query" : "glov",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 3.0
}
}
},
{
"match" : {
"brandNameLowerSyn" : {
"query" : "glov",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 3.0
}
}
},
{
"match" : {
"brandNameLower" : {
"query" : "weighted glov",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 3.0
}
}
},
{
"match" : {
"brandNameLowerSyn" : {
"query" : "weighted glov",
"operator" : "OR",
"prefix_length" : 0,
"max_expansions" : 50,
"fuzzy_transpositions" : true,
"lenient" : false,
"zero_terms_query" : "NONE",
"boost" : 3.0
}
}
},
{
"bool" : {
"must" : [
{
"nested" : {
"query" : {
"bool" : {
"must" : [
{
"term" : {
"categories.categoryNameLower" : {
"value" : "weighted",
"boost" : 4.0
}
}
}
],
"disable_coord" : false,
"adjust_pure_negative" : true,
"boost" : 1.0
org.elasticsearch.transport.RemoteTransportException: [scrpelasticsearchproducts][10.1.0.5:9300][indices:data/read/search[phase/query]]
Caused by: org.elasticsearch.common.breaker.CircuitBreakingException: [request] Data too large, data for [<agg [filter_brand]>] would be larger than limit of [4488796569/4.1gb]
[2020-03-12T16:30:30,401][WARN ][o.e.i.b.request ] [request] New used memory 5089678336 [4.7gb] for data of [<agg [filter_brand]>] would be larger than configured breaker: 4488796569 [4.1gb], breaking
I have tried to google and apply the below settings but it still didn't help.
curl -XPUT localhost:9200/_cluster/settings -d '{
"persistent" : {
"indices.breaker.fielddata.limit" : "40%"
}
}
curl -XPUT 'localhost:9200/_settings' \
-d '{"index.routing.allocation.disable_allocation": false}''
The above changes also didn't help, can any please guide how do we fix this?