Hi,
I've updated my cluster from Elasticsearch 7.17.16 to 8.16.3 and noticed that Kibana is working way slower in ES 8, because most of the query time is spent in HighlightingPhase. The difference is even bigger when using wildcards. Index settings and mappings are the same on both versions. The index is quite small: 8,202,224 docs, 1.36gb storage size.
Mapping:
{
"mappings": {
"_doc": {
"dynamic": "strict",
"properties": {
"alternateUrls": {
"type": "nested",
"properties": {
"alternateDefinitionType": {
"type": "text"
},
"alternateLocale": {
"type": "text"
},
"targetName": {
"type": "text"
},
"targetType": {
"type": "text"
},
"targetValue": {
"type": "text"
}
}
},
"alternateUrlsRawForQuickSearch": {
"type": "text"
},
"boxConditions": {
"type": "nested",
"properties": {
"device": {
"type": "text"
},
"experimentExpectedVariant": {
"type": "text"
},
"experimentId": {
"type": "text"
},
"experimentParameterName": {
"type": "text"
},
"experimentParameterValue": {
"type": "text"
},
"expression": {
"type": "text"
},
"fromDate": {
"type": "date",
"format": "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX"
},
"guestFlag": {
"type": "boolean"
},
"isTrue": {
"type": "boolean"
},
"localeCode": {
"type": "text"
},
"toDate": {
"type": "date",
"format": "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX"
},
"type": {
"type": "text"
},
"userTraitHash": {
"type": "text"
},
"userTraitValue": {
"type": "text"
}
}
},
"boxConditionsRawForQuickSearch": {
"type": "text"
},
"dataSourceAcceptMediaTypes": {
"type": "text"
},
"dataSourceAllowCustomParams": {
"type": "boolean"
},
"dataSourceAllowsPassingPageUrlParams": {
"type": "boolean"
},
"dataSourceCategory": {
"type": "text"
},
"dataSourceDataTypeName": {
"type": "text"
},
"dataSourceDataTypeVersion": {
"type": "version"
},
"dataSourceDescription": {
"type": "text"
},
"dataSourceDomain": {
"type": "text"
},
"dataSourceIcon": {
"type": "text"
},
"dataSourceId": {
"type": "text"
},
"dataSourceInterfaces": {
"type": "text"
},
"dataSourceIsDeprecated": {
"type": "boolean"
},
"dataSourceOAuthMode": {
"type": "text"
},
"dataSourcePassRenderingContext": {
"type": "text"
},
"dataSourceServiceName": {
"type": "text"
},
"dataSourceTimeoutInMillis": {
"type": "integer"
},
"dataSourceTitle": {
"type": "text"
},
"dataSourceUrl": {
"type": "text"
},
"dictionaryKey": {
"type": "text"
},
"dictionaryLanguageCode": {
"type": "text"
},
"dictionaryValue": {
"type": "text"
},
"id": {
"type": "integer"
},
"isPublished": {
"type": "boolean"
},
"name": {
"type": "keyword"
},
"nodeId": {
"type": "text"
},
"pageUrlParameters": {
"type": "nested",
"properties": {
"canonical": {
"type": "boolean"
},
"defaultValue": {
"type": "text"
},
"description": {
"type": "text"
},
"maxValue": {
"type": "double"
},
"minValue": {
"type": "double"
},
"name": {
"type": "text"
},
"required": {
"type": "boolean"
},
"screenParameterName": {
"type": "text"
},
"type": {
"type": "text"
},
"valueMask": {
"type": "text"
}
}
},
"pageUrlParametersRawForQuickSearch": {
"type": "text"
},
"parameters": {
"type": "nested",
"properties": {
"name": {
"type": "text"
},
"sourceOfParameter": {
"type": "text"
},
"type": {
"type": "text"
},
"value": {
"type": "text"
}
}
},
"parametersRawForQuickSearch": {
"type": "text"
},
"prototypeId": {
"type": "text"
},
"prototypeKind": {
"type": "text"
},
"prototypeName": {
"type": "text"
},
"prototypeVersion": {
"type": "version"
},
"publicationRoutes": {
"type": "text"
},
"redirects": {
"type": "nested",
"properties": {
"conditionName": {
"type": "text"
},
"conditionType": {
"type": "text"
},
"conditionValue": {
"type": "text"
},
"locationName": {
"type": "text"
},
"locationType": {
"type": "text"
},
"locationValue": {
"type": "text"
}
}
},
"redirectsRawForQuickSearch": {
"type": "text"
},
"relativeDefinitionUrl": {
"type": "text"
},
"relativeDefinitionUrlForBox": {
"type": "text"
},
"relativeDefinitionUrlForDataSource": {
"type": "text"
},
"siteName": {
"type": "text"
},
"snapshotAuthor": {
"type": "text"
},
"snapshotCreatedAt": {
"type": "date",
"format": "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX"
},
"sourceTab": {
"type": "text"
},
"spaceName": {
"type": "text"
},
"type": {
"type": "text"
}
}
}
}
}
Settings:
"settings": {
"index": {
"routing": {
"allocation": {
"include": {
"_tier_preference": "data_content"
}
}
},
"number_of_shards": "1",
"provided_name": "index-1",
"analysis": {
"analyzer": {
"default": {
"filter": [
"lowercase"
],
"type": "custom",
"tokenizer": "whitespace"
}
}
},
"number_of_replicas": "1",
"uuid": "..",
"version": {
"created": "7171199"
}
}
},
Query:
POST /search-alias/_search
{
"profile": true,
"track_total_hits": true,
"sort": [
{
"_score": {
"order": "desc"
}
}
],
"fields": [
{
"field": "*",
"include_unmapped": true
},
{
"field": "boxConditions.fromDate",
"format": "strict_date_optional_time"
},
{
"field": "boxConditions.toDate",
"format": "strict_date_optional_time"
},
{
"field": "snapshotCreatedAt",
"format": "strict_date_optional_time"
}
],
"size": 500,
"version": true,
"_source": false,
"stored_fields": [
"*"
],
"query": {
"bool": {
"must": [
{
"query_string": {
"query": "search phrase test"
}
}
],
"filter": [],
"should": [],
"must_not": []
}
},
"highlight": {
"pre_tags": [
"@kibana-highlighted-field@"
],
"post_tags": [
"@/kibana-highlighted-field@"
],
"fields": {
"*": {}
},
"fragment_size": 2147483647
},
"runtime_mappings": {},
"script_fields": {}
}
Response ES7:
"hits" : {
"total" : {
"value" : 6557,
"relation" : "eq"
},
...
"type" : "HighlightPhase",
"description" : "",
"time_in_nanos" : 287429894,
"breakdown" : {
"process_count" : 500,
"process" : 287424040,
"next_reader" : 5854,
"next_reader_count" : 8
}
Response ES8:
{
"type": "HighlightPhase",
"description": "",
"time_in_nanos": 1291815787,
"breakdown": {
"process_count": 500,
"process": 1291804079,
"next_reader": 11708,
"next_reader_count": 15
}
},
HighlightingPhase takes 4.5 times longer.
When the query changes to "query": "*search phrase test*", difference is even bigger.
Reponse ES7 2:
{
"type" : "HighlightPhase",
"description" : "",
"time_in_nanos" : 440599218,
"breakdown" : {
"process_count" : 500,
"process" : 440591589,
"next_reader" : 7629,
"next_reader_count" : 9
}
}
Response ES8 2:
{
"type": "HighlightPhase",
"description": "",
"time_in_nanos": 43822082892,
"breakdown": {
"process_count": 500,
"process": 43822062829,
"next_reader": 20063,
"next_reader_count": 14
}
},
Now it takes 100x longer, 440ms on ES7 and 43s on ES8.
Is this a bug or can I do something to fix this performance? Kibana is significantly slower.