I'm doing a terms aggregation over a property on nested documents that match some filter. I'm seeing some weird behavior where the aggregation seems to take a similar amount of time for very different sizes of document sets. Clearing the cache between queries, when filtered down to about 500 documents, it takes around 470ms, and when filtered down to about 400k documents, it takes about 490ms. I would have expected the aggregation to be much quicker over the smaller document set, since the documentation says:
"The context of the execution defines what this document set is (e.g. a top-level aggregation executes within the context of the executed query/filters of the search request)."
Another strange thing is that when running the profiler, many of the "time" measurements are much larger over the large dataset, but this is not reflected when running queries without the profiler. Any ideas about what's going on here?
I'm running Elastic 2.4.4.
The queries I'm using are of the form:
{
"query": {
"term": {
"super_pots": "foo"
}
},
"from": 0,
"size": 0,
"sort": [],
"aggs": {
"nested_properties": {
"nested": {
"path": "custom_property_value"
},
"aggs": {
"filtered_by_attribute": {
"filter": {
"term": {
"custom_property_value.proto_id": "bar"
}
},
"aggs": {
"grouped_by_enum_custom_field": {
"terms": {
"field": "custom_property_value.long_value"
}
}
}
}
}
}
}
}
and the profiles look like:
small document set:
{
"profile": {
"shards": [
{
"id": "[84KrMJh5Q8WQU2lB1c41ZQ][luna_2][194]",
"searches": [
{
"query": [
{
"query_type": "TermQuery",
"lucene": "custom_property_value.proto_id: ~%E|:",
"time": "0.3472080000ms",
"breakdown": {
"match": 0,
"score": 0,
"create_weight": 218361,
"build_scorer": 89981,
"next_doc": 0,
"advance": 38866
}
},
{
"query_type": "BooleanQuery",
"lucene": "+super_pots: ~rcl #ConstantScore(_type:Task)",
"time": "12.52373700ms",
"breakdown": {
"match": 0,
"score": 0,
"create_weight": 351457,
"build_scorer": 4974291,
"next_doc": 1588153,
"advance": 0
},
"children": [
{
"query_type": "TermQuery",
"lucene": "super_pots: ~rcl",
"time": "1.005529000ms",
"breakdown": {
"match": 0,
"score": 0,
"create_weight": 163615,
"build_scorer": 41357,
"next_doc": 800557,
"advance": 0
}
},
{
"query_type": "ConstantScoreQuery",
"lucene": "ConstantScore(_type:Task)",
"time": "4.604307000ms",
"breakdown": {
"match": 0,
"score": 0,
"create_weight": 168259,
"build_scorer": 20690,
"next_doc": 0,
"advance": 2883730
},
"children": [
{
"query_type": "TermQuery",
"lucene": "_type:Task",
"time": "1.531628000ms",
"breakdown": {
"match": 0,
"score": 0,
"create_weight": 163366,
"build_scorer": 11100,
"next_doc": 0,
"advance": 1357162
}
}
]
}
]
}
],
"rewrite_time": 18559,
"collector": [
{
"name": "MultiCollector",
"reason": "search_multi",
"time": "360.3624190ms",
"children": [
{
"name": "TotalHitCountCollector",
"reason": "search_count",
"time": "0.6330810000ms"
},
{
"name": "NestedAggregator: [nested_properties]",
"reason": "aggregation",
"time": "357.8574120ms"
}
]
}
]
}
]
}
]
}
}
big document set:
{
"profile": {
"shards": [
{
"id": "[dkhU9KLORGSrxWhw91XPNg][luna_2][877]",
"searches": [
{
"query": [
{
"query_type": "TermQuery",
"lucene": "custom_property_value.proto_id: )-e[Y!k",
"time": "341.5729560ms",
"breakdown": {
"match": 0,
"score": 0,
"create_weight": 2679454,
"build_scorer": 146813,
"next_doc": 0,
"advance": 338746689
}
},
{
"query_type": "BooleanQuery",
"lucene": "+super_pots: '$wQo #ConstantScore(_type:Task)",
"time": "6185.926116ms",
"breakdown": {
"match": 0,
"score": 0,
"create_weight": 2977196,
"build_scorer": 186133,
"next_doc": 3612754836,
"advance": 0
},
"children": [
{
"query_type": "TermQuery",
"lucene": "super_pots: '$wQo",
"time": "522.9690030ms",
"breakdown": {
"match": 0,
"score": 0,
"create_weight": 2803592,
"build_scorer": 37960,
"next_doc": 520127451,
"advance": 0
}
},
{
"query_type": "ConstantScoreQuery",
"lucene": "ConstantScore(_type:Task)",
"time": "2047.038948ms",
"breakdown": {
"match": 0,
"score": 0,
"create_weight": 136166,
"build_scorer": 20214,
"next_doc": 0,
"advance": 1529017069
},
"children": [
{
"query_type": "TermQuery",
"lucene": "_type:Task",
"time": "517.8654990ms",
"breakdown": {
"match": 0,
"score": 0,
"create_weight": 119880,
"build_scorer": 10677,
"next_doc": 0,
"advance": 517734942
}
}
]
}
]
}
],
"rewrite_time": 48228,
"collector": [
{
"name": "MultiCollector",
"reason": "search_multi",
"time": "3008.855189ms",
"children": [
{
"name": "TotalHitCountCollector",
"reason": "search_count",
"time": "378.1129470ms"
},
{
"name": "NestedAggregator: [nested_properties]",
"reason": "aggregation",
"time": "1497.407875ms"
}
]
}
]
}
]
}
]
}
}