I'm getting an aggregation_execution_exception from a query with elasticsearch server 7.x with a reason of: [nested] nested path [productCategories] is not nested. The equivalent code previously worked in server 5.5.1 with the 5.5.1 Nest dlls. We're attempting to build an upgraded version with the 7.x stack. The server and clients are installed separately from the 5.x servers and clients (we're not upgrading existing instances directly).
As far as I can tell from looking for this error online, it's usually people that mistyped their manually created json for the mappings or a typo in the query call. I'm not sure how I could be performing the same kind of issue with the Nest package generating my query (no json that talks to the server is done by hand or through custom serialization, only Nest). The key point is the claim that the "type: nested" would be missing from the mapping definition, but after using wireshark to capture the info, it's definitely there.3
Also note: in this mapping, there is attributes and if I have that as part of my query before productCategories I get the same error but with the name attributes instead (using kibana to test the swap). Taking both off the query and it runs (but I don't get those aggregates which I need).
Create Index Definition request
(captured with wireshark, extraneous properties that are simple strings and booleans, etc removed to simplify this posting)
PUT http://{host}:9200/product-search-dev-local?pretty=true&include_type_name=true&error_trace=true HTTP/1.1
User-Agent: elasticsearch-net/7.13.2+4bf93bba3d65f6fb8bb67c2955493cba48b624ef (Microsoft Windows 10.0.19043; .NET Framework 4.8.4400.0; Nest)
x-elastic-client-meta: es=7.13.2,net=4.8.0,t=7.13.2,a=1,wr=4.8.0
Content-Length: 12173
{
"mappings": {
"properties": {
"iD": {
"type": "integer"
},
"customKey": {
"analyzer": "product-name-analyzer",
"fields": {
"keyword": {
"analyzer": "product-name-keyword",
"type": "text"
},
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"name": {
"analyzer": "product-name-analyzer",
"fields": {
"keyword": {
"analyzer": "product-name-keyword",
"type": "text"
},
"raw": {
"type": "keyword"
}
},
"type": "text"
},
"attributes": {
"properties": {
"iD": {
"fields": {
"raw": {
"type": "text"
},
"keyword": {
"type": "keyword"
}
},
"type": "text"
},
"key": {
"analyzer": "attribute-name-analyzer",
"fields": {
"raw": {
"analyzer": "attribute-name-keyword",
"type": "text"
},
"keyword": {
"type": "keyword"
}
},
"type": "text"
},
"value": {
"fields": {
"raw": {
"type": "text"
},
"keyword": {
"type": "keyword"
}
},
"type": "text"
},
"uofM": {
"fields": {
"raw": {
"type": "text"
},
"keyword": {
"type": "keyword"
}
},
"type": "text"
},
"sortOrder": {
"type": "integer"
}
},
"type": "nested"
},
"productCategories": {
"properties": {
"categoryName": {
"analyzer": "category-name-analyzer",
"fields": {
"raw": {
"analyzer": "category-name-keyword",
"type": "text"
},
"keyword": {
"type": "keyword"
}
},
"type": "text"
},
"categoryDisplayName": {
"analyzer": "category-name-analyzer",
"fields": {
"raw": {
"analyzer": "category-name-keyword",
"type": "text"
},
"keyword": {
"type": "keyword"
}
},
"type": "text"
},
"categoryParent1Name": {
"analyzer": "category-name-analyzer",
"fields": {
"raw": {
"analyzer": "category-name-keyword",
"type": "text"
},
"keyword": {
"type": "keyword"
}
},
"type": "text"
},
"categoryParent2Name": {
"analyzer": "category-name-analyzer",
"fields": {
"raw": {
"analyzer": "category-name-keyword",
"type": "text"
},
"keyword": {
"type": "keyword"
}
},
"type": "text"
},
"categoryParent3Name": {
"analyzer": "category-name-analyzer",
"fields": {
"raw": {
"analyzer": "category-name-keyword",
"type": "text"
},
"keyword": {
"type": "keyword"
}
},
"type": "text"
},
"categoryParent4Name": {
"analyzer": "category-name-analyzer",
"fields": {
"raw": {
"analyzer": "category-name-keyword",
"type": "text"
},
"keyword": {
"type": "keyword"
}
},
"type": "text"
},
"categoryParent5Name": {
"analyzer": "category-name-analyzer",
"fields": {
"raw": {
"analyzer": "category-name-keyword",
"type": "text"
},
"keyword": {
"type": "keyword"
}
},
"type": "text"
},
"categoryParent6Name": {
"analyzer": "category-name-analyzer",
"fields": {
"raw": {
"analyzer": "category-name-keyword",
"type": "text"
},
"keyword": {
"type": "keyword"
}
},
"type": "text"
}
},
"type": "nested"
}
}
},
"settings": {
"index.number_of_replicas": 0,
"analysis": {
"analyzer": {
"product-name-analyzer": {
"filter": ["product-name-words", "lowercase", "my_stop", "my_snowball", "my_synonyms"],
"tokenizer": "product-name-tokenizer",
"type": "custom"
},
"product-name-suggest-analyzer": {
"filter": ["lowercase"],
"tokenizer": "product-name-path-tokenizer",
"type": "custom"
},
"category-name-analyzer": {
"filter": ["category-name-words", "lowercase", "my_stop", "my_snowball", "my_synonyms"],
"tokenizer": "category-name-tokenizer",
"type": "custom"
},
"attribute-name-analyzer": {
"filter": ["attribute-name-words", "lowercase", "my_stop", "my_snowball", "my_synonyms"],
"tokenizer": "attribute-name-tokenizer",
"type": "custom"
},
"product-name-keyword": {
"filter": ["lowercase", "my_stop", "my_snowball", "my_synonyms"],
"tokenizer": "keyword",
"type": "custom"
},
"category-name-keyword": {
"filter": ["lowercase", "my_stop", "my_snowball", "my_synonyms"],
"tokenizer": "keyword",
"type": "custom"
},
"attribute-name-keyword": {
"filter": ["lowercase", "my_stop", "my_snowball", "my_synonyms"],
"tokenizer": "keyword",
"type": "custom"
},
"synonym-analyzer": {
"filter": ["lowercase", "my_stop", "my_snowball", "my_synonyms"],
"tokenizer": "whitespace",
"type": "custom"
}
},
"filter": {
"product-name-words": {
"generate_number_parts": false,
"generate_word_parts": true,
"preserve_original": true,
"split_on_case_change": true,
"split_on_numerics": true,
"type": "word_delimiter"
},
"category-name-words": {
"generate_number_parts": true,
"generate_word_parts": true,
"preserve_original": true,
"split_on_case_change": true,
"split_on_numerics": false,
"type": "word_delimiter"
},
"attribute-name-words": {
"generate_number_parts": true,
"generate_word_parts": true,
"preserve_original": true,
"split_on_case_change": true,
"split_on_numerics": true,
"type": "word_delimiter"
},
"my_stop": {
"remove_trailing": true,
"stopwords": ["a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", "with"],
"type": "stop"
},
"my_snowball": {
"language": "English",
"type": "snowball"
},
"my_synonyms": {
"format": "solr",
"synonyms": ["i-pod, i pod => ipod", "sea biscuit, sea biscit => seabiscuit"],
"tokenizer": "whitespace",
"type": "synonym"
}
},
"tokenizer": {
"product-name-tokenizer": {
"pattern": "\\W+",
"type": "pattern"
},
"category-name-tokenizer": {
"group": 1,
"pattern": "(?:(.+)(?:\\W*))\\|",
"type": "pattern"
},
"attribute-name-tokenizer": {
"pattern": "\\W+",
"type": "pattern"
},
"product-name-path-tokenizer": {
"delimiter": " ",
"reverse": true,
"type": "path_hierarchy"
}
}
},
"index.number_of_shards": 2
}
}
Populate Index request
(captured with wireshark, abbreviated to one example product and formatted for legibility)
POST http://{host}:9200/product-search-dev-local.2021-09-21.21-33-08/_bulk?pretty=true
User-Agent: elasticsearch-net/7.13.2+4bf93bba3d65f6fb8bb67c2955493cba48b624ef (Microsoft Windows 10.0.19043; .NET Framework 4.8.4400.0; Nest)
x-elastic-client-meta: es=7.13.2,net=4.8.0,t=7.13.2,a=1,wr=4.8.0
Content-Length: 319085 # original length, before trimming for this posting
{"index": {"_id": "1"}}
{
"iD": 1,
"customKey": "432957",
"active": true,
"createdDate": "0001-01-01T00:00:00",
"updatedDate": "2021-07-13T23:42:10.4787939",
"name": "Acer® - Aspire 17.3\" Gaming Laptop",
"attributes": [{
"iD": 2,
"key": "Material",
"value": "Composite",
"uofM": "",
"sortOrder": 10
}, {
"iD": 4,
"key": "Color",
"value": "Black",
"uofM": "",
"sortOrder": 30
}, {
"iD": 5,
"key": "Is Digital Download",
"value": "No",
"uofM": "",
"sortOrder": 40
}
],
"productCategories": [{
"categoryName": "Movies|CAT-1"
}, {
"categoryName": "Electronics|CAT-6"
}, {
"categoryName": "Computer|Electronics_Computer"
}
]
}
Query request
(captured with wireshark, an alias that clips the date stamp off the index was added prior to this call)
POST http://{host}:9200//product-search-dev-local/_search?pretty=true&typed_keys=true
User-Agent: elasticsearch-net/7.13.2+4bf93bba3d65f6fb8bb67c2955493cba48b624ef (Microsoft Windows 10.0.19043; .NET Framework 4.8.4400.0; Nest)
x-elastic-client-meta: es=7.13.2,net=4.8.0,t=7.13.2,a=1,wr=4.8.0
Content-Length: 2235
Connection: Keep-Alive
{
"aggs": {
"product-categories": {
"aggs": {
"category-names-parent": {
"aggs": {
"category-names-5": {
"aggs": {
"category-names-4": {
"aggs": {
"category-names-3": {
"aggs": {
"category-names-2": {
"aggs": {
"category-names-1": {
"aggs": {
"category-names-0": {
"terms": {
"field": "productCategories.categoryName.keyword",
"missing": "N/A",
"size": 500
}
}
},
"terms": {
"field": "productCategories.categoryParent1Name.keyword",
"missing": "N/A",
"size": 500
}
}
},
"terms": {
"field": "productCategories.categoryParent2Name.keyword",
"missing": "N/A",
"size": 500
}
}
},
"terms": {
"field": "productCategories.categoryParent3Name.keyword",
"missing": "N/A",
"size": 500
}
}
},
"terms": {
"field": "productCategories.categoryParent4Name.keyword",
"missing": "N/A",
"size": 500
}
}
},
"terms": {
"field": "productCategories.categoryParent5Name.keyword",
"missing": "N/A",
"size": 500
}
}
},
"terms": {
"field": "productCategories.categoryParent6Name.keyword",
"missing": "N/A",
"size": 500
}
}
},
"nested": {
"path": "productCategories"
}
},
"product-price-ranges": {
"aggs": {
"product-price-ranges-max": {
"max": {
"field": "finalPrice",
"missing": 0.0
}
},
"product-price-ranges-min": {
"min": {
"field": "finalPrice",
"missing": 0.0
}
}
},
"range": {
"field": "finalPrice",
"ranges": [{
"key": "< $50",
"to": 50.0
}, {
"from": 50.0,
"key": "$50 - $99.99",
"to": 99.9999
}, {
"from": 100.0,
"key": "$100 - $199.99",
"to": 199.9999
}, {
"from": 200.0,
"key": "$200 - $299.99",
"to": 299.9999
}, {
"from": 300.0,
"key": "$300 - $399.99",
"to": 399.9999
}, {
"from": 400.0,
"key": "$400 - $499.99",
"to": 499.9999
}, {
"from": 500.0,
"key": "$500 - $999.99",
"to": 999.9999
}, {
"from": 1000.0,
"key": "$1000 - $1999.99",
"to": 1999.9999
}, {
"from": 2000.0,
"key": "$2000 - $2999.99",
"to": 2999.9999
}, {
"from": 3000.0,
"key": "$3000 - $3999.99",
"to": 3999.9999
}, {
"from": 4000.0,
"key": "$4000 - $4999.99",
"to": 4999.9999
}, {
"from": 5000.0,
"key": "$5000 +"
}
]
}
},
"product-attributes": {
"aggs": {
" attribute-keys": {
"aggs": {
" attribute-values": {
"terms": {
"field": "attributes.value.keyword",
"missing": "N/A",
"size": 100
}
}
},
"terms": {
"field": "attributes.key.keyword",
"missing": "N/A",
"size": 100
}
}
},
"nested": {
"path": "attributes"
}
}
},
"from": 0,
"query": {
"function_score": {
"functions": [{
"field_value_factor": {
"field": "totalPurchasedQuantity",
"factor": 0.01
}
}
],
"max_boost": 10.0
}
},
"size": 10000,
"sort": [{
"_score": {
"order": "desc"
}
}, {
"sortOrder": {
"missing": "_last",
"order": "asc"
}
}
]
}
The response
{
"error" : {
"root_cause" : [
{
"type" : "aggregation_execution_exception",
"reason" : "[nested] nested path [productCategories] is not nested"
}
],
"type" : "search_phase_execution_exception",
"reason" : "all shards failed",
"phase" : "query",
"grouped" : true,
"failed_shards" : [
{
"shard" : 0,
"index" : "product-search-dev-local.2021-09-21.21-53-47",
"node" : "E-30Rx7qRCqJhvvnjCVTmg",
"reason" : {
"type" : "aggregation_execution_exception",
"reason" : "[nested] nested path [productCategories] is not nested"
}
}
]
},
"status" : 500
}
I can get into the Nest C# setup if necessary, but I'm hoping someone can point out where the problem is, or some feature change that happened between 5 and 7 which I'm not accounting for and is causing it.
I'm sorry that I had to flatten the json blocks in order to get under the character limit for this forum.