We recently did an upgrade from 7.4 to 7.10 and have seen over a 900% increase in search latency.
Before the upgrade our average search latency was about 0.5ms and after the upgrade our average search latency is now at 5ms.
My only guess is that it is due to a change in the version of Lucene from 7.4 to 7.10?
Three sample queries that used to return in microseconds but now return in milliseconds. Any help would be greatly appreciated! Thank you.
Query 1
[2022-06-24T11:56:11,755][WARN ][index.search.slowlog.query] [0c45479e9393989bde99a9dc672293ec] [event-2022.06.22][0] took[623.6micros], took_millis[0], total_hits[40 hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[15], source[{
"from": 0,
"size": 5000,
"query": {
"bool": {
"must": [
{
"match": {
"routingId": {
"query": "419460a5-d4b1-4e74-a3b9-1c18f13XXXXX",
"operator": "OR",
"prefix_length": 0,
"max_expansions": 50,
"fuzzy_transpositions": true,
"lenient": false,
"zero_terms_query": "NONE",
"auto_generate_synonyms_phrase_query": true,
"boost": 1
}
}
},
{
"bool": {
"must": [
{
"bool": {
"should": [
{
"match": {
"type": {
"query": "DEVICE_ZONE_RUN_COMPLETED_EVENT",
"operator": "OR",
"prefix_length": 0,
"max_expansions": 50,
"fuzzy_transpositions": true,
"lenient": false,
"zero_terms_query": "NONE",
"auto_generate_synonyms_phrase_query": true,
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
{
"range": {
"timestamp": {
"from": null,
"to": null,
"include_lower": true,
"include_upper": true,
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"sort": [
{
"timestamp": {
"order": "asc"
}
}
]
}], id[],
Query 2
[2022-06-24T11:56:11,624][WARN ][index.search.slowlog.query] [0c45479e9393989bde99a9dc672293ec] [event-2022.06.22][0] took[460.4micros], took_millis[0], total_hits[1 hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[31], source[{
"from": 0,
"size": 1,
"query": {
"bool": {
"must": [
{
"match": {
"routingId": {
"query": "f4fc5463-15b4-4cb5-9d68-6ea0388XXXXX",
"operator": "OR",
"prefix_length": 0,
"max_expansions": 50,
"fuzzy_transpositions": true,
"lenient": false,
"zero_terms_query": "NONE",
"auto_generate_synonyms_phrase_query": true,
"boost": 1
}
}
},
{
"bool": {
"must": [
{
"bool": {
"should": [
{
"match": {
"type": {
"query": "SCHEDULE_COMPLETED_EVENT",
"operator": "OR",
"prefix_length": 0,
"max_expansions": 50,
"fuzzy_transpositions": true,
"lenient": false,
"zero_terms_query": "NONE",
"auto_generate_synonyms_phrase_query": true,
"boost": 1
}
}
},
{
"match": {
"type": {
"query": "SCHEDULE_STOPPED_EVENT",
"operator": "OR",
"prefix_length": 0,
"max_expansions": 50,
"fuzzy_transpositions": true,
"lenient": false,
"zero_terms_query": "NONE",
"auto_generate_synonyms_phrase_query": true,
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
{
"range": {
"timestamp": {
"from": null,
"to": null,
"include_lower": true,
"include_upper": true,
"boost": 1
}
}
},
{
"match": {
"scheduleId": {
"query": "e189edcb-717d-4e12-8f12-66425b5XXXXX",
"operator": "OR",
"prefix_length": 0,
"max_expansions": 50,
"fuzzy_transpositions": true,
"lenient": false,
"zero_terms_query": "NONE",
"auto_generate_synonyms_phrase_query": true,
"boost": 1
}
}
},
{
"match": {
"executionType": {
"query": "SCHEDULED",
"operator": "OR",
"prefix_length": 0,
"max_expansions": 50,
"fuzzy_transpositions": true,
"lenient": false,
"zero_terms_query": "NONE",
"auto_generate_synonyms_phrase_query": true,
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"sort": [
{
"timestamp": {
"order": "desc"
}
}
]
}], id[],
Query 3
[2022-06-24T11:56:11,621][WARN ][index.search.slowlog.query] [0c45479e9393989bde99a9dc672293ec] [event-2022.06.22][0] took[273.6micros], took_millis[0], total_hits[16 hits], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[16], source[{
"from": 0,
"size": 5000,
"query": {
"bool": {
"must": [
{
"match": {
"routingId": {
"query": "409ead84-481a-4b37-b893-6a6c323XXXXX",
"operator": "OR",
"prefix_length": 0,
"max_expansions": 50,
"fuzzy_transpositions": true,
"lenient": false,
"zero_terms_query": "NONE",
"auto_generate_synonyms_phrase_query": true,
"boost": 1
}
}
},
{
"bool": {
"must": [
{
"bool": {
"should": [
{
"match": {
"type": {
"query": "MOISTURE_DATA_EVENT",
"operator": "OR",
"prefix_length": 0,
"max_expansions": 50,
"fuzzy_transpositions": true,
"lenient": false,
"zero_terms_query": "NONE",
"auto_generate_synonyms_phrase_query": true,
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
{
"range": {
"timestamp": {
"from": null,
"to": null,
"include_lower": true,
"include_upper": true,
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"sort": [
{
"timestamp": {
"order": "desc"
}
}
]
}], id[], ```