Bulk update taking time on updating nested properties

Elastic Stack Elasticsearch
1

I am running bulk query on elastic search for updating user's nested properties.

Below is the query which I am executing to update user's properties. It updates around 200 properties of single user.
It takes around 5000 mili seconds

Machine configuration on Amazon Aws
Master : 1, c4.large
Data Nodes : 2 c4.2xlarge

POST /INDEX_NAME/user/_bulk
{"update":{"_id":"55950ed51d4213241f7c38b4","type":"user","index":"jal","routing":2,"_retry_on_conflict":5,"refresh":true}}
{"lang":"mvel","script":"foreach (item : ctx._source.category) { if (item[ 'key' ] == key) { if(item[ 'name' ]==null){item[ 'name' ] =name;} } }","params":{"key":"26069_purchased","name":"26069"}}
{"update":{"_id":"55950ed51d4213241f7c38b4","type":"user","index":"jal","routing":2,"_retry_on_conflict":5,"refresh":true}}
{"lang":"mvel","script":"foreach (item : ctx._source.category) { if (item[ 'key' ] == key) { if(item[ 'action' ]==null){item[ 'action' ] =action;} } }","params":{"key":"26069_purchased","action":"purchased"}}
{"update":{"_id":"55950ed51d4213241f7c38b4","type":"user","index":"jal","routing":2,"_retry_on_conflict":5,"refresh":true}}
{"lang":"mvel","script":"foreach (item : ctx._source.category) { if (item[ 'key' ] == key) {if(item[ 'price' ]==null){item[ 'price' ]=price;}else{ item[ 'price' ]=item[ 'price' ]+ price;} } }","params":{"key":"26069_purchased","price":0}}
{"update":{"_id":"55950ed51d4213241f7c38b4","type":"user","index":"jal","routing":2,"_retry_on_conflict":5,"refresh":true}}
{"lang":"mvel","script":"foreach (item : ctx._source.category) { if (item[ 'key' ] == key) {if(item[ 'count' ]==null){item[ 'count' ]=count;}else{ item[ 'count' ]=item[ 'count' ]+ count;} } }","params":{"key":"26069_purchased","count":1}}
{"update":{"_id":"55950ed51d4213241f7c38b4","type":"user","index":"jal","routing":2,"_retry_on_conflict":5,"refresh":true}}
{"lang":"mvel","script":"foreach (item : ctx._source.category) { if (item[ 'key' ] == key) {if(item[ 'last_date' ]==null){item[ 'last_date' ]=last_date;}else{ item[ 'last_date' ] = last_date; } }}","params":{"key":"26069_purchased","last_date":"2015-06-20T15:37:09"}}
{"update":{"_id":"55950ed51d4213241f7c38b4","type":"user","index":"jal","routing":2,"_retry_on_conflict":5,"refresh":true}}
{"lang":"mvel","script":"foreach (item : ctx._source.category) { if (item[ 'key' ] == key) {if(item[ 'without_promo' ]==null){item[ 'without_promo' ]=without_promo;}else{ item[ 'without_promo' ]=item[ 'without_promo' ]+ without_promo;} } }","params":{"key":"26069_purchased","without_promo":1}}

2

When you update a nested doc it needs to do a delete of the original and then reindex the new one.
Given you are doing this for a single doc that's a lot of delete+index it needs to do.

Why not just roll these into a single update?

3

i will change my update in to single update by doing this.But there is no change. it still taking more than 3000 ms .

{"update":{"_id":"hr324hdie93hdjdo391","type":"user","index":"jal","routing":2,"_retry_on_conflict":5}}
{"lang":"mvel","script":"if(ctx._source.containsKey("category")){i=0; io=0; foreach (item : ctx._source.category){i++; if (item[ 'key' ] == v_123_purchased){ io=1}} if(io==0){ctx._source.category.add(p_123_purchased);}}else{ctx._source.category=c_123_purchased;}if(ctx._source.containsKey("category")){i=0; io=0; foreach (item : ctx._source.category){i++; if (item[ 'key' ] == v_233_purchased){ io=1}} if(io==0){ctx._source.category.add(p_233_purchased);}}else{ctx._source.category=c_233_purchased;}foreach (item : ctx._source.category) { if (item[ 'key' ] == key_123_purchased) { if(item[ 'name' ]==null){item[ 'name' ] =name_123_purchased;} } }foreach (item : ctx._source.category) { if (item[ 'key' ] == key_123_purchased) { if(item[ 'action' ]==null){item[ 'action' ] =action_123_purchased;} } }foreach (item : ctx._source.category) { if (item[ 'key' ] == key_123_purchased) {if(item[ 'price' ]==null){item[ 'price' ]=price_123_purchased;}else{ item[ 'price' ]=item[ 'price' ]+ price_123_purchased;} } }foreach (item : ctx._source.category) { if (item[ 'key' ] == key_123_purchased) {if(item[ 'count' ]==null){item[ 'count' ]=count_123_purchased;}else{ item[ 'count' ]=item[ 'count' ]+ count_123_purchased;} } }foreach (item : ctx._source.category) { if (item[ 'key' ] == key_123_purchased) {if(item[ 'last_date' ]==null){item[ 'last_date' ]=last_date_123_purchased;}else{ item[ 'last_date' ] = last_date_123_purchased; } }}foreach (item : ctx._source.category) { if (item[ 'key' ] == key_123_purchased) {if(item[ 'with_promo' ]==null){item[ 'with_promo' ]=with_promo_123_purchased;}else{ item[ 'with_promo' ]=item[ 'with_promo' ]+ with_promo_123_purchased;} } }foreach (item : ctx._source.category) { if (item[ 'key' ] == key_233_purchased) { if(item[ 'name' ]==null){item[ 'name' ] =name_233_purchased;} } }foreach (item : ctx._source.category) { if (item[ 'key' ] == key_233_purchased) { if(item[ 'action' ]==null){item[ 'action' ] =action_233_purchased;} } }foreach (item : ctx._source.category) { if (item[ 'key' ] == key_233_purchased) {if(item[ 'price' ]==null){item[ 'price' ]=price_233_purchased;}else{ item[ 'price' ]=item[ 'price' ]+ price_233_purchased;} } }foreach (item : ctx._source.category) { if (item[ 'key' ] == key_233_purchased) {if(item[ 'count' ]==null){item[ 'count' ]=count_233_purchased;}else{ item[ 'count' ]=item[ 'count' ]+ count_233_purchased;} } }foreach (item : ctx._source.category) { if (item[ 'key' ] == key_233_purchased) {if(item[ 'last_date' ]==null){item[ 'last_date' ]=last_date_233_purchased;}else{ item[ 'last_date' ] = last_date_233_purchased; } }}foreach (item : ctx._source.category) { if (item[ 'key' ] == key_233_purchased) {if(item[ 'with_promo' ]==null){item[ 'with_promo' ]=with_promo_233_purchased;}else{ item[ 'with_promo' ]=item[ 'with_promo' ]+ with_promo_233_purchased;} } }","params":{"p_123_purchased":{"key":"123_purchased"},"c_123_purchased":[{"key":"123_purchased"}],"v_123_purchased":"123_purchased","p_233_purchased":{"key":"233_purchased"},"c_233_purchased":[{"key":"233_purchased"}],"v_233_purchased":"233_purchased","key_123_purchased":"123_purchased","name_123_purchased":123,"action_123_purchased":"purchased","price_123_purchased":20,"count_123_purchased":1,"last_date_123_purchased":"2015-07-04T07:11:03","with_promo_123_purchased":1,"key_233_purchased":"233_purchased","name_233_purchased":233,"action_233_purchased":"purchased","price_233_purchased":20,"count_233_purchased":1,"last_date_233_purchased":"2015-07-04T07:11:03","with_promo_233_purchased":1}}

Here is my Node status

_plugin HQ nodediagnostics.png757×1344 179 KB

4

Then the problem is likely to be your heavy use of scripting, not sure what else can be done here.

5

You should IMO create a script in config dir and use it or at least use parameters because I think you script have to be compiled for each single line you send which is an heavy operation.

6

Hii david can you please give me any example that how to create script in config file. Or any simple method to update nested object . I just want to update property of nested obj by four ways append , increment , ignore ,replace with new value .
My nested Obj lokking like this

{
               "_userId": "hr324hdie93hdjdo391",
               "_shardId": 2,
               "brand": [
                  {
                     "price": 140,
                     "name": 23,
                     "count": 7,
                     "without_promo": 7,
                     "action": "purchased",
                     "key": "23_purchased",
                     "last_date": "2015-07-04T10:10:25"
                  }
               ],
               "product": [
                  {
                     "price": 140,
                     "with_promo": 7,
                     "name": 28,
                     "count": 7,
                     "action": "purchased",
                     "key": "28_purchased",
                     "last_date": "2015-07-04T10:10:25"
                  },
                  {
                     "price": 40,
                     "with_promo": 2,
                     "name": 29,
                     "count": 2,
                     "action": "purchased",
                     "key": "29_purchased",
                     "last_date": "2015-07-04T06:25:30"
                  }
               ],
               "category": [
                  {
                     "price": 140,
                     "with_promo": 7,
                     "name": 123,
                     "count": 7,
                     "action": "purchased",
                     "key": "123_purchased",
                     "last_date": "2015-07-04T10:10:25"
                  },
                  {
                     "price": 40,
                     "with_promo": 2,
                     "name": 124,
                     "count": 2,
                     "action": "purchased",
                     "key": "124_purchased",
                     "last_date": "2015-07-04T06:25:30"
                  }
               ]
            }
1 Like