Very slow queries stop words and synonyms after upgrade to 6.7.0 from 6.2.3

Elastic Stack Elasticsearch
1

Removing the stop word or synonym, the query is fast. Removing the stop word, removes spanNear([SpanGap(:1), from the query.

ES version 6.7.

"took" : 9913,

(Note: artificial intelligence has a synonym to ai as well)
GET csdev_data/_search
{
"query": {
"match_phrase": {
"bookText_en": "introduction to artificial intelligence"
}
}
}

"description" : "+spanNear([bookText_en:introduct, spanOr([spanNear([SpanGap(:1), bookText_en:ai], 0, true), spanNear([SpanGap(:1), bookText_en:artifici, bookText_en:intellig], 0, true)])], 0, true) #DocValuesFieldExistsQuery [field=_primary_term]",

Without stop word, also ES 6.7

"took" : 21,

GET csdev_data/_search
{
"query": {
"match_phrase": {
"bookText_en": "introduction artificial intelligence"
}
}
}

"description" : "spanNear([bookText_en:introduct, spanOr([bookText_en:ai, spanNear([bookText_en:artifici, bookText_en:intellig], 0, true)])], 0, true)",
2

Can you create a small reproduction, where you create an index, a small mapping, index a sample document and the search, so that this can be reproduced locally?

Thank you!

3

I'm having some trouble trying to make a small reproduction of this. The problem is on a field that contains entire contents of a book.

Does it help to show the profile output of the same query against basically the same documents (~30K documents) but one is ES 6.7.0 and the other 6.2.3?

GET csdev_data/_search
{
  "size": 0, 
  "profile": "true", 
  "query": {
    "match_phrase": {
      "bookText_en": "introduction to artificial intelligence"
    }
  }
}
4

ES 6.2.3
{
"took": 82,
"timed_out": false,
"_shards": {
"total": 3,
"successful": 3,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 4,
"max_score": 0,
"hits":
},
"profile": {
"shards": [
{
"id": "[CN8nQzugSmOq5rg6ctlPZg][csint_data_v15][0]",
"searches": [
{
"query": [
{
"type": "BooleanQuery",
"description": "+spanNear([bookText_en:introduct, spanOr([bookText_en:ai, spanNear([bookText_en:artifici, bookText_en:intellig], 0, true)])], 0, true) #DocValuesFieldExistsQuery [field=_primary_term]",
"time_in_nanos": 43309678,
"breakdown": {
"score": 0,
"build_scorer_count": 24,
"match_count": 782,
"create_weight": 216013,
"next_doc": 14802861,
"match": 24502034,
"create_weight_count": 1,
"next_doc_count": 872,
"score_count": 0,
"build_scorer": 3787091,
"advance": 0,
"advance_count": 0
},
"children": [
{
"type": "SpanNearQuery",
"description": "spanNear([bookText_en:introduct, spanOr([bookText_en:ai, spanNear([bookText_en:artifici, bookText_en:intellig], 0, true)])], 0, true)",
"time_in_nanos": 38723939,
"breakdown": {
"score": 0,
"build_scorer_count": 28,
"match_count": 782,
"create_weight": 180961,
"next_doc": 14051599,
"match": 24222115,
"create_weight_count": 1,
"next_doc_count": 872,
"score_count": 0,
"build_scorer": 267581,
"advance": 0,
"advance_count": 0
}
},
{
"type": "DocValuesFieldExistsQuery",
"description": "DocValuesFieldExistsQuery [field=_primary_term]",
"time_in_nanos": 207743,
"breakdown": {
"score": 0,
"build_scorer_count": 12,
"match_count": 0,
"create_weight": 1179,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 20376,
"advance": 185303,
"advance_count": 872
}
}
]
}
],
"rewrite_time": 19856,
"collector": [
{
"name": "CancellableCollector",
"reason": "search_cancelled",
"time_in_nanos": 74409,
"children": [
{
"name": "TotalHitCountCollector",
"reason": "search_count",
"time_in_nanos": 17104
}
]
}
]
}
],
"aggregations":
},
{
"id": "[Lapq3tHxRmWfCwrP9ukilw][csint_data_v15][2]",
"searches": [
{
"query": [
{
"type": "BooleanQuery",
"description": "+spanNear([bookText_en:introduct, spanOr([bookText_en:ai, spanNear([bookText_en:artifici, bookText_en:intellig], 0, true)])], 0, true) #DocValuesFieldExistsQuery [field=_primary_term]",
"time_in_nanos": 25566860,
"breakdown": {
"score": 0,
"build_scorer_count": 24,
"match_count": 770,
"create_weight": 8087160,
"next_doc": 10716785,
"match": 6120972,
"create_weight_count": 1,
"next_doc_count": 773,
"score_count": 0,
"build_scorer": 640375,
"advance": 0,
"advance_count": 0
},
"children": [
{
"type": "SpanNearQuery",
"description": "spanNear([bookText_en:introduct, spanOr([bookText_en:ai, spanNear([bookText_en:artifici, bookText_en:intellig], 0, true)])], 0, true)",
"time_in_nanos": 24055670,
"breakdown": {
"score": 0,
"build_scorer_count": 27,
"match_count": 770,
"create_weight": 7992780,
"next_doc": 10057005,
"match": 5813617,
"create_weight_count": 1,
"next_doc_count": 773,
"score_count": 0,
"build_scorer": 190697,
"advance": 0,
"advance_count": 0
}
},
{
"type": "DocValuesFieldExistsQuery",
"description": "DocValuesFieldExistsQuery [field=_primary_term]",
"time_in_nanos": 185859,
"breakdown": {
"score": 0,
"build_scorer_count": 9,
"match_count": 0,
"create_weight": 1565,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 14904,
"advance": 168607,
"advance_count": 773
}
}
]
}
],
"rewrite_time": 21766,
"collector": [
{
"name": "CancellableCollector",
"reason": "search_cancelled",
"time_in_nanos": 60706,
"children": [
{
"name": "TotalHitCountCollector",
"reason": "search_count",
"time_in_nanos": 26649
}
]
}
]
}
],
"aggregations":
},
{
"id": "[XLgiMReITxSavM4wd21JLA][csint_data_v15][1]",
"searches": [
{
"query": [
{
"type": "BooleanQuery",
"description": "+spanNear([bookText_en:introduct, spanOr([bookText_en:ai, spanNear([bookText_en:artifici, bookText_en:intellig], 0, true)])], 0, true) #DocValuesFieldExistsQuery [field=_primary_term]",
"time_in_nanos": 77788229,
"breakdown": {
"score": 0,
"build_scorer_count": 25,
"match_count": 785,
"create_weight": 22249467,
"next_doc": 25406443,
"match": 29165883,
"create_weight_count": 1,
"next_doc_count": 791,
"score_count": 0,
"build_scorer": 964834,
"advance": 0,
"advance_count": 0
},
"children": [
{
"type": "SpanNearQuery",
"description": "spanNear([bookText_en:introduct, spanOr([bookText_en:ai, spanNear([bookText_en:artifici, bookText_en:intellig], 0, true)])], 0, true)",
"time_in_nanos": 74740193,
"breakdown": {
"score": 0,
"build_scorer_count": 29,
"match_count": 785,
"create_weight": 22187615,
"next_doc": 23760863,
"match": 28495509,
"create_weight_count": 1,
"next_doc_count": 791,
"score_count": 0,
"build_scorer": 294600,
"advance": 0,
"advance_count": 0
}
},
{
"type": "DocValuesFieldExistsQuery",
"description": "DocValuesFieldExistsQuery [field=_primary_term]",
"time_in_nanos": 551579,
"breakdown": {
"score": 0,
"build_scorer_count": 12,
"match_count": 0,
"create_weight": 3418,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 27665,
"advance": 519692,
"advance_count": 791
}
}
]
}
],
"rewrite_time": 18247,
"collector": [
{
"name": "CancellableCollector",
"reason": "search_cancelled",
"time_in_nanos": 81169,
"children": [
{
"name": "TotalHitCountCollector",
"reason": "search_count",
"time_in_nanos": 22986
}
]
}
]
}
],
"aggregations":
}
]
}
}

5

ES 6.7.0
{
"took" : 9872,
"timed_out" : false,
"_shards" : {
"total" : 3,
"successful" : 3,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : 13,
"max_score" : 0.0,
"hits" :
},
"profile" : {
"shards" : [
{
"id" : "[PFNuMigHQju3-iCpySBZoQ][csdev_data_v15][0]",
"searches" : [
{
"query" : [
{
"type" : "BooleanQuery",
"description" : "+spanNear([bookText_en:introduct, spanOr([spanNear([SpanGap(:1), bookText_en:ai], 0, true), spanNear([SpanGap(:1), bookText_en:artifici, bookText_en:intellig], 0, true)])], 0, true) #DocValuesFieldExistsQuery [field=_primary_term]",
"time_in_nanos" : 6399032201,
"breakdown" : {
"score" : 0,
"build_scorer_count" : 29,
"match_count" : 782,
"create_weight" : 7585635,
"next_doc" : 2953095988,
"match" : 3437787305,
"create_weight_count" : 1,
"next_doc_count" : 785,
"score_count" : 0,
"build_scorer" : 561676,
"advance" : 0,
"advance_count" : 0
},
"children" : [
{
"type" : "SpanNearQuery",
"description" : "spanNear([bookText_en:introduct, spanOr([spanNear([SpanGap(:1), bookText_en:ai], 0, true), spanNear([SpanGap(:1), bookText_en:artifici, bookText_en:intellig], 0, true)])], 0, true)",
"time_in_nanos" : 6396366746,
"breakdown" : {
"score" : 0,
"build_scorer_count" : 32,
"match_count" : 782,
"create_weight" : 7555591,
"next_doc" : 2951370890,
"match" : 3437246149,
"create_weight_count" : 1,
"next_doc_count" : 785,
"score_count" : 0,
"build_scorer" : 192516,
"advance" : 0,
"advance_count" : 0
}
},
{
"type" : "DocValuesFieldExistsQuery",
"description" : "DocValuesFieldExistsQuery [field=_primary_term]",
"time_in_nanos" : 277071,
"breakdown" : {
"score" : 0,
"build_scorer_count" : 9,
"match_count" : 0,
"create_weight" : 835,
"next_doc" : 0,
"match" : 0,
"create_weight_count" : 1,
"next_doc_count" : 0,
"score_count" : 0,
"build_scorer" : 18165,
"advance" : 257276,
"advance_count" : 785
}
}
]
}
],
"rewrite_time" : 18123,
"collector" : [
{
"name" : "CancellableCollector",
"reason" : "search_cancelled",
"time_in_nanos" : 44478,
"children" : [
{
"name" : "TotalHitCountCollector",
"reason" : "search_count",
"time_in_nanos" : 14827
}
]
}
]
}
],
"aggregations" :
},
{
"id" : "[jSy3A2FBQ6GrTSmy1dTDbg][csdev_data_v15][1]",
"searches" : [
{
"query" : [
{
"type" : "BooleanQuery",
"description" : "+spanNear([bookText_en:introduct, spanOr([spanNear([SpanGap(:1), bookText_en:ai], 0, true), spanNear([SpanGap(:1), bookText_en:artifici, bookText_en:intellig], 0, true)])], 0, true) #DocValuesFieldExistsQuery [field=_primary_term]",
"time_in_nanos" : 9611793748,
"breakdown" : {
"score" : 0,
"build_scorer_count" : 42,
"match_count" : 785,
"create_weight" : 9710004,
"next_doc" : 4149982022,
"match" : 5451568563,
"create_weight_count" : 1,
"next_doc_count" : 790,
"score_count" : 0,
"build_scorer" : 531541,
"advance" : 0,
"advance_count" : 0
},
"children" : [
{
"type" : "SpanNearQuery",
"description" : "spanNear([bookText_en:introduct, spanOr([spanNear([SpanGap(:1), bookText_en:ai], 0, true), spanNear([SpanGap(:1), bookText_en:artifici, bookText_en:intellig], 0, true)])], 0, true)",
"time_in_nanos" : 9609939564,
"breakdown" : {
"score" : 0,
"build_scorer_count" : 45,
"match_count" : 785,
"create_weight" : 9683597,
"next_doc" : 4149041426,
"match" : 5451038293,
"create_weight_count" : 1,
"next_doc_count" : 790,
"score_count" : 0,
"build_scorer" : 174627,
"advance" : 0,
"advance_count" : 0
}
},
{
"type" : "DocValuesFieldExistsQuery",
"description" : "DocValuesFieldExistsQuery [field=_primary_term]",
"time_in_nanos" : 316534,
"breakdown" : {
"score" : 0,
"build_scorer_count" : 9,
"match_count" : 0,
"create_weight" : 890,
"next_doc" : 0,
"match" : 0,
"create_weight_count" : 1,
"next_doc_count" : 0,
"score_count" : 0,
"build_scorer" : 80752,
"advance" : 234092,
"advance_count" : 790
}
}
]
}
],
"rewrite_time" : 15634,
"collector" : [
{
"name" : "CancellableCollector",
"reason" : "search_cancelled",
"time_in_nanos" : 46242,
"children" : [
{
"name" : "TotalHitCountCollector",
"reason" : "search_count",
"time_in_nanos" : 14930
}
]
}
]
}
],
"aggregations" :
},
{
"id" : "[xNarOaDASjar-h_g3RNTWg][csdev_data_v15][2]",
"searches" : [
{
"query" : [
{
"type" : "BooleanQuery",
"description" : "+spanNear([bookText_en:introduct, spanOr([spanNear([SpanGap(:1), bookText_en:ai], 0, true), spanNear([SpanGap(:1), bookText_en:artifici, bookText_en:intellig], 0, true)])], 0, true) #DocValuesFieldExistsQuery [field=_primary_term]",
"time_in_nanos" : 10368790260,
"breakdown" : {
"score" : 0,
"build_scorer_count" : 29,
"match_count" : 770,
"create_weight" : 4821359,
"next_doc" : 5039558415,
"match" : 5320928957,
"create_weight_count" : 1,
"next_doc_count" : 771,
"score_count" : 0,
"build_scorer" : 3479958,
"advance" : 0,
"advance_count" : 0
},
"children" : [
{
"type" : "SpanNearQuery",
"description" : "spanNear([bookText_en:introduct, spanOr([spanNear([SpanGap(:1), bookText_en:ai], 0, true), spanNear([SpanGap(:1), bookText_en:artifici, bookText_en:intellig], 0, true)])], 0, true)",
"time_in_nanos" : 10366796157,
"breakdown" : {
"score" : 0,
"build_scorer_count" : 30,
"match_count" : 770,
"create_weight" : 4796952,
"next_doc" : 5038455998,
"match" : 5320280183,
"create_weight_count" : 1,
"next_doc_count" : 771,
"score_count" : 0,
"build_scorer" : 3261452,
"advance" : 0,
"advance_count" : 0
}
},
{
"type" : "DocValuesFieldExistsQuery",
"description" : "DocValuesFieldExistsQuery [field=_primary_term]",
"time_in_nanos" : 275398,
"breakdown" : {
"score" : 0,
"build_scorer_count" : 3,
"match_count" : 0,
"create_weight" : 837,
"next_doc" : 0,
"match" : 0,
"create_weight_count" : 1,
"next_doc_count" : 0,
"score_count" : 0,
"build_scorer" : 4419,
"advance" : 269367,
"advance_count" : 771
}
}
]
}
],
"rewrite_time" : 25479,
"collector" : [
{
"name" : "CancellableCollector",
"reason" : "search_cancelled",
"time_in_nanos" : 34514,
"children" : [
{
"name" : "TotalHitCountCollector",
"reason" : "search_count",
"time_in_nanos" : 10947
}
]
}
]
}
],
"aggregations" :
}
]
}
}

6

This topic was automatically closed 28 days after the last reply. New replies are no longer allowed.