Hello, we're switching from ES 5.5 to 7.12. After applying it works perfect for about 10-12 hrs: and then we get CPU loading: 100%
Even simple queries: GET /products/_search?size=0 works for 5-6 seconds.
And it returns to the normal state in a couple of seconds when we switch back.
ES version: 7.12
number_of_shards: 18
number_of_replicas: 2
nodes number: 9
index size: 267 Gb
RAM: 32GB
Xms16g
Xmx16g
CPUs: 8
Number of documents: 9581754
50-80 documents indexing per second (no bulk used)
mapping
{
"products" : {
"mappings" : {
"dynamic" : "strict",
"properties" : {
"categories" : {
"type" : "long"
},
"categoryId" : {
"type" : "long"
},
"ean" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword"
}
}
},
"features" : {
"type" : "nested",
"dynamic" : "strict",
"properties" : {
"categoryFeatureId" : {
"type" : "long"
},
"value" : {
"type" : "keyword",
"eager_global_ordinals" : true
}
}
},
"localizedData" : {
"type" : "nested",
"dynamic" : "strict",
"properties" : {
"category" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword"
}
}
},
"languageShortCode" : {
"type" : "keyword"
},
"lifeCycle" : {
"type" : "date",
"format" : "yyyy-MM-dd HH:mm:ss"
},
"supplier" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword"
}
}
},
"title" : {
"type" : "text"
},
}
},
"mpn" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword"
}
},
"analyzer" : "autoCompleteAnalyzer"
},
"numericalFeatures" : {
"type" : "nested",
"dynamic" : "strict",
"properties" : {
"categoryFeatureId" : {
"type" : "long"
},
"value" : {
"type" : "float"
}
}
},
"price" : {
"type" : "float"
},
"supplierId" : {
"type" : "long"
}
}
}
}
}
two typical queries used for calculating filters and for getting filtered results
{
"aggs": {
"all": {
"global": {},
"aggs": {
"main": {
"filter": {
"bool": {
"must": [
{
"nested": {
"path": "localizedData",
"query": {
"bool": {
"must": [
{
"term": {
"localizedData.languageShortCode": {
"value": "en"
}
}
},
{
"bool": {
"should": [
{
"range": {
"localizedData.lifeCycle": {
"lte": "2021-04-23 08:23:32"
}
}
},
{
"bool": {
"must_not": {
"exists": {
"field": "localizedData.lifeCycle"
}
}
}
}
]
}
}
]
}
}
}
}
]
}
},
"aggs": {
"general": {
"filter": {
"bool": {
"must": [
{
"nested": {
"path": "features",
"query": {
"bool": {
"must": [
{
"match": {
"features.categoryFeatureId": "36303"
}
},
{
"match": {
"features.value": "y"
}
}
]
}
}
}
},
{
"bool": {
"should": [
{
"term": {
"categoryId": {
"value": 151,
"boost": 1000
}
}
},
{
"term": {
"categories": {
"value": 151
}
}
}
]
}
}
]
}
},
"aggs": {
"price": {
"stats": {
"field": "price"
}
},
"cat": {
"terms": {
"field": "categories",
"size": 30
}
},
"man": {
"terms": {
"field": "supplierId",
"size": 30
}
},
"nested_features": {
"nested": {
"path": "features"
},
"aggs": {
"featureId": {
"terms": {
"field": "features.categoryFeatureId",
"size": 100
},
"aggs": {
"featureVal": {
"terms": {
"field": "features.value",
"size": 100
}
}
}
}
}
},
"nested_numericalFeatures": {
"nested": {
"path": "numericalFeatures"
},
"aggs": {
"featureId": {
"terms": {
"field": "numericalFeatures.categoryFeatureId",
"size": 100
},
"aggs": {
"featureVal": {
"stats": {
"field": "numericalFeatures.value"
}
}
}
}
}
}
}
}
}
}
}
}
}
}
{
"query": {
"function_score": {
"query": {
"bool": {
"must": [
{
"nested": {
"path": "localizedData",
"query": {
"bool": {
"must": [
{
"term": {
"localizedData.languageShortCode": {
"value": "en"
}
}
},
{
"bool": {
"should": [
{
"range": {
"localizedData.lifeCycle": {
"lte": "2021-04-23 08:26:21"
}
}
},
{
"bool": {
"must_not": {
"exists": {
"field": "localizedData.lifeCycle"
}
}
}
}
]
}
}
]
}
}
}
},
{
"nested": {
"path": "features",
"query": {
"bool": {
"must": [
{
"match": {
"features.categoryFeatureId": "36303"
}
},
{
"match": {
"features.value": "y"
}
}
]
}
}
}
},
{
"bool": {
"should": [
{
"term": {
"categoryId": {
"value": 151,
"boost": 1000
}
}
},
{
"term": {
"categories": {
"value": 151
}
}
}
]
}
}
]
}
},
"functions": [
{
"filter": {
"nested": {
"path": "features",
"query": {
"exists": {
"field": "features.categoryFeatureId"
}
}
}
},
"weight": 50
},
{
"filter": {
"match": {
"quality": 2
}
},
"weight": 20
},
{
"filter": {
"match": {
"quality": 1
}
},
"weight": 10
}
],
"boost_mode": "sum"
}
}
}
show stats uploaded
