While upgrading elasticsearch from 5.6 to 7.9, we noticed that the DocValuesFieldExistsQuery is getting called for all child fields of a field, causing wait times up to a minute. Is there a way to prevent triggering DocValuesFieldExistsQuery call for all subfields when "exists": "embedded.files" is used? Current implementation behaves as if "exists": "embedded.files.*" was requested. We can't disable doc_values for subfields in favor of _field_names because our faceted search depends on aggregations on these subfields.
Query
{
"query": {
"bool": {
"filter": [
{
"bool": {
"must": [
{
"terms": {
"principals_allowed.view": [
"system.Everyone"
]
}
},
{
"terms": {
"embedded.@type": [
"Experiment"
]
}
}
]
}
}
]
}
},
"post_filter": {
"bool": {
"must": [
{
"terms": {
"embedded.assay_title": [
"TF ChIP-seq"
]
}
},
{
"terms": {
"embedded.@type": [
"Experiment"
]
}
},
{
"exists": {
"field": "embedded.files"
}
}
]
}
},
"aggs": {
"Data Type": {
"filter": {
"bool": {
"must": [
{
"terms": {
"embedded.assay_title": [
"TF ChIP-seq"
]
}
},
{
"exists": {
"field": "embedded.files"
}
}
]
}
} ...
5.6 profile
{
"type": "ConstantScoreQuery",
"description": "ConstantScore(_field_names:embedded.files)",
"time": "0.3931280000ms",
"time_in_nanos": 393128,
"breakdown": {
"score": 0,
"build_scorer_count": 57,
"match_count": 0,
"create_weight": 7076,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 186433,
"advance": 198625,
"advance_count": 936
},
"children": [
{
"type": "TermQuery",
"description": "_field_names:embedded.files",
"time": "0.2634930000ms",
"time_in_nanos": 263493,
"breakdown": {
"score": 0,
"build_scorer_count": 57,
"match_count": 0,
"create_weight": 2241,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 143295,
"advance": 116963,
"advance_count": 936
}
}
]
}
]
},
7.9 Profile:
{
"type" : "BooleanQuery",
"description" : "DocValuesFieldExistsQuery [field=embedded.files.submitted_by.schema_version] DocValuesFieldExistsQuery [field=embedded.files.quality_metrics.lab] DocValuesFieldExistsQuery [field=embedded.files.quality_metrics.% of reads unmapped: other] DocValuesFieldExistsQuery [field=embedded.files.quality_metrics.snRNA.raw] DocValuesFieldExistsQuery ... (a few thousand more)
Mapping
"embedded": {
"properties": {
"@id": {
"type": "keyword"
},
"files": {
"properties": {
"@id": {
"type": "keyword"
},
"@type": {
"type": "keyword"
},
"accession": {
"type": "keyword",
"copy_to": [
"_all"
]
},
"aliases": {
"type": "keyword"
},
"alternate_accessions": {
"type": "keyword",
"copy_to": [
"_all"
]
},
"analysis_step_version": {
"properties": {
"@id": {
"type": "keyword"
},
"@type": {
"type": "keyword"
},
"aliases": {
"type": "keyword"
},
"analysis_step": {
"properties": {
"@id": {
"type": "keyword"
},
"@type": {
"type": "keyword"
},
"aliases": {
"type": "keyword" ...