With ES/Kibana 6.4.0.
We have 16TB+/150+Bil indices in ES, with high cardinality fields such as uid.
If turn on autocomplete feature, then input uid: in search bar, kibana would send this query to ES:
{
"size": 0,
"timeout": "1s",
"terminate_after": 100000,
"query": {
"match_all": {
"boost": 1.0
}
},
"aggregations": {
"suggestions": {
"terms": {
"field": "uid",
"size": 10,
"shard_size": 10,
"min_doc_count": 1,
"shard_min_doc_count": 0,
"show_term_doc_count_error": false,
"execution_hint": "map",
"order": [
{
"_count": "desc"
},
{
"_key": "asc"
}
],
"include": ".*"
}
}
}
}
After 10+min, It will load 160GB+ fielddata to memory, seems loading fielddata can't terminate property.
And this is top hot threads:
10/10 snapshots sharing following 41 elements
org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:168)
org.apache.lucene.index.OrdinalMap.build(OrdinalMap.java:147)
org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder.build(GlobalOrdinalsBuilder.java:65)
org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:130)
org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.localGlobalDirect(SortedSetDVOrdinalsIndexFieldData.java:47)
org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.lambda$load$1(IndicesFieldDataCache.java:165)
org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache$$Lambda$3267/1059313702.load(Unknown Source)
org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache$IndexFieldCache.load(IndicesFieldDataCache.java:162)
org.elasticsearch.index.fielddata.plain.SortedSetDVOrdinalsIndexFieldData.loadGlobal(SortedSetDVOrdinalsIndexFieldData.java:118)
org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals$FieldData.globalOrdinalsValues(ValuesSource.java:151)
org.elasticsearch.search.aggregations.support.ValuesSource$Bytes$WithOrdinals.globalMaxOrd(ValuesSource.java:124)
org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.getMaxOrd(TermsAggregatorFactory.java:215)
org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.doCreateInternal(TermsAggregatorFactory.java:137)
org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory.createInternal(ValuesSourceAggregatorFactory.java:55)
org.elasticsearch.search.aggregations.AggregatorFactory.create(AggregatorFactory.java:216)
org.elasticsearch.search.aggregations.AggregatorFactories.createTopLevelAggregators(AggregatorFactories.java:216)
org.elasticsearch.search.aggregations.AggregationPhase.preProcess(AggregationPhase.java:55)
org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:105)
org.elasticsearch.indices.IndicesService.lambda$loadIntoContext$17(IndicesService.java:1184)
org.elasticsearch.indices.IndicesService$$Lambda$3192/791757691.accept(Unknown Source)
org.elasticsearch.indices.IndicesService.lambda$cacheShardLevelResult$18(IndicesService.java:1237)
org.elasticsearch.indices.IndicesService$$Lambda$3194/1117148458.get(Unknown Source)
org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:160)
org.elasticsearch.indices.IndicesRequestCache$Loader.load(IndicesRequestCache.java:143)
org.elasticsearch.common.cache.Cache.computeIfAbsent(Cache.java:433)
org.elasticsearch.indices.IndicesRequestCache.getOrCompute(IndicesRequestCache.java:116)
org.elasticsearch.indices.IndicesService.cacheShardLevelResult(IndicesService.java:1243)
org.elasticsearch.indices.IndicesService.loadIntoContext(IndicesService.java:1183)
org.elasticsearch.search.SearchService.loadOrExecuteQueryPhase(SearchService.java:322)
org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:357)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:333)
org.elasticsearch.search.SearchService$2.onResponse(SearchService.java:329)
org.elasticsearch.search.SearchService$3.doRun(SearchService.java:1019)
org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:723)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)
org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
java.lang.Thread.run(Thread.java:748)
Why build OrdinalMap cost too much CPU times, and terminate_after doesn't become effective.